From a383d4bfeae1b2957472722aba8d103be3619e21 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 08:55:57 +0100 Subject: [PATCH 01/60] GPU: Add option to print USS --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 37 +++++++++++++++++++ GPU/GPUTracking/Base/GPUReconstruction.h | 4 ++ GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 3 ++ .../Base/GPUReconstructionCPUKernels.h | 3 ++ GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + 5 files changed, 48 insertions(+) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index 3e7509d8fad73..d1e96d6df9288 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -934,6 +934,9 @@ void GPUReconstruction::PopNonPersistentMemory(RecoStep step, uint64_t tag, cons mNonPersistentIndividualDirectAllocations.resize(std::get<3>(mNonPersistentMemoryStack.back())); mNonPersistentMemoryStack.pop_back(); } + if (!proc && GetProcessingSettings().queryHostMemoryUSS) { // Outside of the lock above + QueryHostMemoryUSS(("pop " + qTag2Str(tag)).c_str()); + } } void GPUReconstruction::BlockStackedMemory(GPUReconstruction* rec) @@ -995,6 +998,40 @@ void GPUReconstruction::PrintMemoryMax() printf("Maximum Memory Allocation: Host %'zu / Device %'zu\n", mHostMemoryUsedMax, mDeviceMemoryUsedMax); } +void GPUReconstruction::QueryHostMemoryUSS(const char* where) +{ +#ifdef __linux__ + size_t uss = 0; + if (FILE* fp = fopen("/proc/self/smaps_rollup", "r")) { + char line[256]; + while (fgets(line, sizeof(line), fp)) { + size_t val; + if (sscanf(line, "Private_Clean: %zu kB", &val) == 1 || sscanf(line, "Private_Dirty: %zu kB", &val) == 1 || sscanf(line, "Private_Hugetlb: %zu kB", &val) == 1) { + uss += val * 1024; + } + } + fclose(fp); + } + stdspinlock spinlock(mMemoryMutex); + if (uss > mHostMemoryUSSMax) { + mHostMemoryUSSMax = uss; + mHostMemoryUSSMaxWhere = where; + } + if (GetProcessingSettings().queryHostMemoryUSS >= 2) { + printf("Host Memory USS after %s: %'zu\n", where, uss); + } +#endif +} + +void GPUReconstruction::PrintHostMemoryUSSMax() +{ +#ifdef __linux__ + printf("Maximum Host Memory USS: %'zu (after %s)\n", mHostMemoryUSSMax, mHostMemoryUSSMaxWhere.c_str()); +#else + printf("Host Memory USS query not supported on this platform\n"); +#endif +} + void GPUReconstruction::PrintMemoryOverview() { if (GetProcessingSettings().memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) { diff --git a/GPU/GPUTracking/Base/GPUReconstruction.h b/GPU/GPUTracking/Base/GPUReconstruction.h index eb2653fdfb66d..3a2d644ac71f1 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.h +++ b/GPU/GPUTracking/Base/GPUReconstruction.h @@ -195,6 +195,8 @@ class GPUReconstruction void PrintMemoryStatistics(); void PrintMemoryOverview(); void PrintMemoryMax(); + void QueryHostMemoryUSS(const char* where); // With queryHostMemoryUSS: query the USS of the process and keep track of the maximum + void PrintHostMemoryUSSMax(); void SetMemoryExternalInput(int16_t res, void* ptr); GPUMemorySizeScalers* MemoryScalers() { return mMemoryScalers.get(); } @@ -350,6 +352,8 @@ class GPUReconstruction void* mHostMemoryPoolBlocked = nullptr; // Ptr to end of pool size_t mHostMemorySize = 0; // Size of host memory buffer size_t mHostMemoryUsedMax = 0; // Maximum host memory size used over time + size_t mHostMemoryUSSMax = 0; // Maximum USS of the process seen by QueryHostMemoryUSS + std::string mHostMemoryUSSMaxWhere; // and where it was seen void* mDeviceMemoryBase = nullptr; // Same for device ... void* mDeviceMemoryPermanent = nullptr; // ... void* mDeviceMemoryPool = nullptr; // ... diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 9ee6fae1e0fd9..be8b5eed49156 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -333,6 +333,9 @@ int32_t GPUReconstructionCPU::RunChains() mTimerTotal.Reset(); } + if (GetProcessingSettings().queryHostMemoryUSS) { + PrintHostMemoryUSSMax(); + } if (GetProcessingSettings().memoryStat) { PrintMemoryStatistics(); } else if (GetProcessingSettings().debugLevel >= 2) { diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPUKernels.h b/GPU/GPUTracking/Base/GPUReconstructionCPUKernels.h index 7970fbe753ec8..3e330cd82ef6c 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPUKernels.h +++ b/GPU/GPUTracking/Base/GPUReconstructionCPUKernels.h @@ -76,6 +76,9 @@ inline void GPUReconstructionCPU::runKernelInterface(krnlSetup&& setup, Args con if (GPUDebug(GetKernelName(), stream, GetProcessingSettings().serializeGPU & 1)) { throw std::runtime_error("kernel failure"); } + if (GetProcessingSettings().queryHostMemoryUSS) { + QueryHostMemoryUSS(GetKernelName()); + } if (GetProcessingSettings().debugLevel >= 1) { if (t) { if (deviceTimerTime != 0.) { diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index e323f5f0fc6e5..194fb3302d604 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -415,6 +415,7 @@ AddOption(debugOnFailureMaxSize, uint32_t, 0, "", 0, "Max size of existing dumps AddOption(debugOnFailureDirectory, std::string, ".", "", 0, "Target folder for debug / dump") AddOption(ROOTDumpFile, std::string, "gpudebug.root", "", 0, "File name for ROOT dump (default gpudebug.root)") AddOption(memoryStat, bool, false, "", 0, "Print memory statistics") +AddOption(queryHostMemoryUSS, int8_t, 0, "", 0, "Query the unique set size (private resident memory) of the process after every kernel and after every pop of the non-persistent memory stack, and print the maximum (2: print every query). Slow, Linux only", def(1)) AddVariable(runsStandalone, bool, false) AddVariable(eventDisplay, o2::gpu::GPUDisplayFrontendInterface*, nullptr) AddSubConfig(GPUSettingsProcessingRTC, rtc) From 76bb7637a1bd71d07df1777f84e37cf764fbc1e4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 11:36:21 +0100 Subject: [PATCH 02/60] GPU CMake: Mark GPUCA_DETERMINISTIC_MODE=1 invalid --- dependencies/FindO2GPU.cmake | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 18472b0c19a40..a8a2924872bf0 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -10,7 +10,7 @@ # or submit itself to any jurisdiction. # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources -# FindO2GPU.cmake Version 24 +# FindO2GPU.cmake Version 25 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual) set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908) @@ -146,12 +146,13 @@ if(DEFINED ENV{O2_OVERRIDE_GPUCA_DETERMINISTIC_MODE}) set(GPUCA_DETERMINISTIC_MODE $ENV{O2_OVERRIDE_GPUCA_DETERMINISTIC_MODE}) endif() # set(GPUCA_DETERMINISTIC_MODE WHOLEO2) # Override -set(GPUCA_DETERMINISTIC_MODE_MAP_OFF 0) -set(GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH 1) # No -ffast-math and similar compile flags for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_OPTO2 2) # In addition, -O2 optimization on host for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_GPU 3) # In addition, GPUCA_DETERMINISTIC_MODE define for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_ON 3) # Synonym for GPU -set(GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2 4) # As GPU but for whole O2 code +set(GPUCA_DETERMINISTIC_MODE_MAP_OFF 0) # Off - best performance +set(GPUCA_DETERMINISTIC_MODE_MAP_INVALID 1) # Invalid value, so people do not use 1 accidentally +set(GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH 2) # No -ffast-math and similar compile flags for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_OPTO2 3) # In addition, -O2 optimization on host for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_GPU 4) # In addition, GPUCA_DETERMINISTIC_MODE define for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_ON 4) # Synonym for GPU +set(GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2 5) # As GPU but for whole O2 code if(NOT DEFINED GPUCA_DETERMINISTIC_MODE) set(GPUCA_DETERMINISTIC_MODE 0) elseif(NOT GPUCA_DETERMINISTIC_MODE MATCHES "^[0-9]+$") @@ -160,6 +161,9 @@ elseif(NOT GPUCA_DETERMINISTIC_MODE MATCHES "^[0-9]+$") endif() set(GPUCA_DETERMINISTIC_MODE ${GPUCA_DETERMINISTIC_MODE_MAP_${GPUCA_DETERMINISTIC_MODE}}) endif() +if(GPUCA_DETERMINISTIC_MODE STREQUAL 1) + message(FATAL_ERROR "GPUCA_DETERMINISTIC_MODE = 1 is invalid and not supported, use ON or GPU or see FindO2GPU.cmake for details") +endif() if(GPUCA_DETERMINISTIC_NO_FTZ) set(GPUCA_CXX_DENORMALS_FLAGS "") set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=false") From d0cef01e33646d924411d6bdd3b7a7e8d26d493b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 15:52:07 +0200 Subject: [PATCH 03/60] GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Global/GPUChainTrackingMerger.cxx | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 49 +++++-------------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 7 files changed, 17 insertions(+), 53 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 194fb3302d604..b61c413521cb0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -152,7 +152,6 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails (=2 means retry in same kernel, =1 for separate kernel") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 528a848acf389..fe1d06d27f602 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,9 +235,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); - if (param().rec.tpc.retryRefit == 1) { - runKernel(GetGridAuto(0), -1); - } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index cf8ea3c4f4675..233bdcf3e7d0b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,8 +402,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers - computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); // Reducing mNMaxTracks for mLoopData / mRetryRefitIds does not save memory, since the other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index eefaf6fb74497..a18ad20fc6e88 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,7 +67,6 @@ class GPUTPCGMMerger : public GPUProcessor static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { - GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -129,7 +128,6 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } - GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -299,7 +297,6 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; - uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index bf01ea234a645..a7df0d279c08f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,10 +21,9 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { + const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7b70d94b9cfc5..74c9651d33dde 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); - prop.SetSeedingErrors(!(refit && attempt == 0)); + prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); @@ -104,7 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - int32_t goodRows = 0; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); if (crossCE) { @@ -178,11 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (lastRow == 255 || CAMath::Abs((int32_t)lastRow - (int32_t)cluster.row) > 5 || lastSector != cluster.sector || (param.rec.tpc.trackFitRejectMode < 0 && -nMissed <= param.rec.tpc.trackFitRejectMode)) { - goodRows = 0; - } else { - goodRows++; - } if (retValProp == 0) { lastRow = cluster.row; lastSector = cluster.sector; @@ -227,17 +221,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { int8_t rejectChi2 = 0; - if (attempt == 0) { - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } else { - rejectChi2 = allowChangeClusters && goodRows > 5; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; } } @@ -904,7 +894,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, int32_t attempt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -919,27 +909,10 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, attempt, constants::MAX_SIN_PHI, track); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); CADEBUG(printf("Finished Fit Track %d\n", iTrk)); CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); - if (!ok && attempt == 0 && merger->Param().rec.tpc.retryRefit) { - for (uint32_t i = 0; i < track.NClusters(); i++) { - merger->Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - CADEBUG(printf("Track rejected, marking for retry\n")); - if (merger->Param().rec.tpc.retryRefit == 2) { - nTrackHits = track.NClusters(); - NTolerated = 0; // Clusters not fit but tollerated for track length cut - t = track.Param(); - Alpha = track.Alpha(); - ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, 1, constants::MAX_SIN_PHI, track); - } else { - uint32_t nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1u); - merger->RetryRefitIds()[nRefit] = iTrk; - return; - } - } if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = 1.e-4f; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index c9d8f2017af52..fffa3f5180653 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,7 +200,7 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, int32_t attempt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { From 9466d5c19ecb674613010e6324d578d7a1f90278 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 21:34:57 +0200 Subject: [PATCH 04/60] GPU TPC: Add possibility to run last way of TPC fit in separate kernel to rebuild track ion between --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 8 ++++ .../Global/GPUChainTrackingMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 46 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/kernels.cmake | 2 +- 8 files changed, 47 insertions(+), 26 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b61c413521cb0..f24bb3d918d65 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -135,6 +135,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Dif AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 8a22e2c03ea47..8ab663b5f791a 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -266,6 +266,14 @@ bool GPUChainTracking::ValidateSettings() GPUError("Cannot do error interpolation with NWays < 3!"); return false; } + if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { + GPUError("Need error interpolation to rebuild tracks during fit"); + return false; + } + if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { + GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); + return false; + } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { GPUError("configured max time bin exceeds 256 orbits"); return false; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index fe1d06d27f602..7b712831c2a23 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -234,7 +234,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + if (param().rec.tpc.rebuildTrackInFit) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index a7df0d279c08f..22d8b159e9b74 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -19,11 +19,11 @@ using namespace o2::gpu; template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) +GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index d703f1b890443..b4f823aa11ed8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -36,7 +36,7 @@ class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 74c9651d33dde..f1abf9ecd866d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -60,7 +60,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors) { + if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { for (int32_t i = 0; i < N; i++) { interpolation.hit[i].errorY = -1; } @@ -70,18 +70,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const int32_t maxN = N; int32_t ihitStart = 0; float covYYUpd = 0.f; - float lastUpdateX = -1.f; - uint8_t lastRow = 255; - uint8_t lastSector = 255; float deltaZ = 0.f; - for (int32_t iWay = 0; iWay < nWays; iWay++) { + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { - StoreOuter(&track.OuterParam(), prop.GetAlpha()); + StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! } int32_t resetT0 = initResetT0(); @@ -95,12 +92,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - lastUpdateX = -1; + uint8_t lastRow = 255; + uint8_t lastSector = 255; + float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -110,9 +109,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ lastSector = clusters[ihit].sector; } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (finalOutInFit && !(clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject)) { + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; } continue; @@ -327,6 +329,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } else { deltaZ = 0.f; } + + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + Alpha = prop.GetAlpha(); + if (ihitStart != 0) { + MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + } + return true; + } } ConstrainSinPhi(); @@ -894,7 +904,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -908,21 +918,21 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); - CADEBUG(printf("Finished Fit Track %d\n", iTrk)); - CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { - t.QPt() = 1.e-4f; + t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); track.SetOK(ok); - track.SetNClustersFitted(nTrackHits); track.Param() = t; track.Alpha() = Alpha; + if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + track.SetNClustersFitted(nTrackHits); + } // if (track.OK()) merger->DebugRefitMergedTrack(track); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index fffa3f5180653..9a9b22bbd2f7a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,8 +201,6 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); - GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { if (mP[2] > limit) { diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index ed155788b0bef..138eb8a6339db 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,7 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From 61d485ce605d1758b80534479bb8ba005942b16d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 23 Sep 2025 17:58:33 +0200 Subject: [PATCH 05/60] GPU TPC: Search hit closest to interpolated point in track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 26 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 131 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 1 + .../GPUTPCTrackletConstructor.cxx | 4 +- 8 files changed, 149 insertions(+), 26 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index f24bb3d918d65..09f36bacbf13c 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -136,6 +136,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Fin AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 7b712831c2a23..d22778ea41b63 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -96,7 +96,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) uint32_t numBlocks = (!mRec->IsGPU() || doGPU) ? BlockCount() : 1; GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; - GPUTPCGMMerger& MergerShadowAll = doGPU ? processorsShadow()->tpcMerger : Merger; const int32_t outputStream = OutputStream(); if (GetProcessingSettings().debugLevel >= 2) { GPUInfo("Running TPC Merger"); @@ -142,7 +141,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge within Sectors runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeWithinSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); @@ -150,23 +149,23 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge between sectors - transport to the middle of the sector and rotate vertically to the border on the left / right runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeAtMidRow, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - transport to the left / right edge of the sector and rotate horizontally - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - use original track param - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeWithOriginalParameters, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); if (GetProcessingSettings().mergerSanityCheck) { @@ -208,8 +207,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (maxId > Merger.NMaxClusters()) { throw std::runtime_error("mNMaxClusters too small"); } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.SharedCount(), maxId * sizeof(*MergerShadowAll.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -234,6 +233,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } + if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); @@ -268,13 +270,13 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) throw std::runtime_error("QA Scratch buffer exceeded"); } } - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadowAll.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadow.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); waitEvent = nullptr; if (param().dodEdxEnabled) { - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadowAll.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } - GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadowAll.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadowAll.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 233bdcf3e7d0b..9b7e2c0256f8a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -403,6 +403,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -1656,7 +1657,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; } nHits += nTrackHits; } @@ -1964,7 +1965,7 @@ GPUd() void GPUTPCGMMerger::MergeLoopersInit(int32_t nBlocks, int32_t nThreads, for (uint32_t k = 0;k < trk.NClusters();k++) { float xx, yy, zz; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[mClusters[trk.FirstClusterRef() + k].num]; - GetConstantMem()->calibObjects.fastTransformHelper->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); + GetConstantMem()->calibObjects.fastTransform->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); float sa2, ca2; CAMath::SinCos(Param().Alpha(mClusters[trk.FirstClusterRef() + k].sector), sa2, ca2); float cx = ca2 * xx - sa2 * yy; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index a18ad20fc6e88..2afca0512783d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -78,10 +78,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { + struct trackCluster { // TODO: Reduce size of this struct! uint32_t id; uint8_t row; uint8_t sector; + float error; }; struct tmpSort { @@ -125,6 +126,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } + GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -272,6 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + trackCluster* mClusterCandidates = nullptr; GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index f1abf9ecd866d..57a2de5979fb1 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,9 +47,10 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -72,9 +73,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { @@ -135,7 +136,26 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, allowChangeClusters) == -1) { + if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { + while (true) { + if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { + break; + } + } + const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (interHit.id == 1) { + nMissed++; + nMissed2++; + continue; + } + const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; + merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (interHit.id - 2 == clusters[ihit].num) { + clusterState = clusters[ihit].state; + } else { + clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -155,7 +175,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // clang-format off CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); // clang-format on - if (allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); @@ -200,7 +220,102 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } float uncorrectedY = -1e6f; - if (allowChangeClusters) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); + GPUglobalref() const cahit2* hits = tracker.HitData(row); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); + const auto& inter = interpolation.hit[ihit]; + if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float y0 = row.Grid().YMin(); + const float stepY = row.HstepY(); + const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = row.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + float uncorrectedZ; + merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = row.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + } + if (nCandidates == 0) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } @@ -320,7 +435,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ break; // bad chi2 for the whole track, stop the fit } } - if (finalOutInFit && !(merger->Param().rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -918,7 +1033,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 9a9b22bbd2f7a..b287e5a717edf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index cb0c8d4a76fdb..7c9047202417b 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -167,6 +167,7 @@ class GPUTPCTracker : public GPUProcessor } return ((int32_t)weight); } + GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } GPUd() void MaximizeHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.MaximizeHitWeight(row, hitIndex, weight); } GPUd() void SetHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.SetHitWeight(row, hitIndex, weight); } GPUd() int32_t HitWeight(const GPUTPCRow& row, int32_t hitIndex) const { return mData.HitWeight(row, hitIndex); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..86979c29006ae 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -199,7 +199,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float sErr2 = tracker.Param().GetSystematicClusterErrorIFC2(x, tParam.GetY(), tParam.GetZ(), tracker.ISector() >= 18); err2Y += sErr2; err2Z += sErr2; - const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; + const float kFactor = tracker.GetChiSeedFactor(); float sy2 = kFactor * (tParam.Err2Y() + err2Y); float sz2 = kFactor * (tParam.Err2Z() + err2Z); if (sy2 > tracker.Param().rec.tpc.hitSearchArea2) { @@ -307,7 +307,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; const float maxWindow2 = tracker.Param().rec.tpc.hitSearchArea2; const float sy2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Y() + err2Y)); - const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); + const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); // TODO: Double check if the factors are enough, errors could be larger with flags int32_t bin, ny, nz; row.Grid().GetBinArea(yUncorrected, zUncorrected + tParam.ZOffset(), CAMath::Sqrt(sy2), CAMath::Sqrt(sz2), bin, ny, nz); From 40369d78558fdf2d06ca126bf89b3407fa14ec54 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 5 Oct 2025 18:31:00 +0200 Subject: [PATCH 06/60] GPU TPC: Fix index for hit interpolation in rows with multiple clusters --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 57a2de5979fb1..c5484cc8f523c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -219,13 +219,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - const auto& inter = interpolation.hit[ihit]; if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); const float y0 = row.Grid().YMin(); @@ -357,10 +357,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && interpolation.hit[ihit].errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { rejectChi2 = GPUTPCGMPropagator::rejectDirect; } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &interpolation.hit[ihit], err2Y, err2Z, deltaZ); + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); } } @@ -370,7 +370,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); } GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, interpolation.hit[ihit], rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); }); } // clang-format off From 0232089b77d66e544af543a2b44484cb32be5daf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 29 Sep 2025 22:01:44 +0200 Subject: [PATCH 07/60] GPU TPC: Implement ambiguity solving for rebuild tracks cluster association --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- .../Definitions/Parameters/GPUParameters.csv | 5 + .../GPUChainTrackingDebugAndProfiling.cxx | 6 +- .../Global/GPUChainTrackingMerger.cxx | 22 +- GPU/GPUTracking/Global/GPUErrorCodes.h | 20 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 220 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 27 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 31 +++ GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 13 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +-- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 5 +- .../Standalone/Benchmark/standalone.cxx | 2 +- GPU/GPUTracking/kernels.cmake | 5 + GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 15 files changed, 322 insertions(+), 79 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 09f36bacbf13c..a7b96b692cb35 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -137,12 +137,13 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following, 8: mirroring)") +AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 8e41173cee40a..5892f4acfe9c4 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -52,6 +52,11 @@ GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,256,,256,25 GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,256,,256,256, +GPUTPCGMMergerHitWeights_prepare,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_compute,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolve1,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolve2,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolveShared,256,,,,,,,,,,,,,,,,,, GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,256,,256,256, diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index 7659d19693777..e45aeb4ba9246 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -187,8 +187,8 @@ void GPUChainTracking::PrintMemoryRelations() GPUInfo("MEMREL SectorTrackHits NCl %d NTrkH %d", processors()->tpcTrackers[i].NHitsTotal(), *processors()->tpcTrackers[i].NTrackHits()); } if (processors()->tpcMerger.Memory()) { - GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTracks()); - GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTrackClusters()); + GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTracks()); + GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTrackClusters()); } } @@ -217,7 +217,7 @@ void GPUChainTracking::PrintKernelDebugOutput() void GPUChainTracking::PrintOutputStat() { int32_t nTracks = 0, nAttachedClusters = 0, nAttachedClustersFitted = 0, nAdjacentClusters = 0; - uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NMaxClusters(); + uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NClusters(); if (GetProcessingSettings().createO2Output > 1) { nTracks = mIOPtrs.nOutputTracksTPCO2; nAttachedClusters = mIOPtrs.nMergedTrackHits; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index d22778ea41b63..7f393446b2744 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -203,12 +203,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } - uint32_t maxId = Merger.NMaxClusters(); - if (maxId > Merger.NMaxClusters()) { - throw std::runtime_error("mNMaxClusters too small"); - } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -238,6 +234,16 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + for (int32_t i = 0; i < param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + if (i + 1 < param().rec.tpc.rebuildTrackInFitClusterCandidates) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + } + } + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); @@ -276,7 +282,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); @@ -366,7 +372,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 2) { - GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NClusters()); + GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NSectorHits()); } return 0; } diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index a4921f478b107..2080548c7775c 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -33,16 +33,16 @@ GPUCA_ERROR_CODE(13, ERROR_SECTORDATA_HITINROW_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(17, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(18, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER - +GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 9b7e2c0256f8a..d9b139b73ab7e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -398,12 +398,17 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackIDs, GPUTPCGeometry::NSECTORS * mNMaxSingleSectorTracks); // UnpackResetIds - RefitSectorTracks - UnpackSectorGlobal memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 - computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); + computePointerWithAlignment(mem, mSharedCount, mNClusters); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 + computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { + computePointerWithAlignment(mem, mSharedCount2, mNClusters); + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); + computePointerWithAlignment(mem, mHitWeights, mNClusters); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -437,14 +442,14 @@ void* GPUTPCGMMerger::SetPointersOutput(void* mem) } } computePointerWithAlignment(mem, mClusters, mNMaxMergedTrackClusters); - computePointerWithAlignment(mem, mClusterAttachment, mNMaxClusters); + computePointerWithAlignment(mem, mClusterAttachment, mNClusters); return mem; } void* GPUTPCGMMerger::SetPointersOutputState(void* mem) { if ((mRec->GetRecoSteps() & gpudatatypes::RecoStep::Refit) || mRec->GetProcessingSettings().outputSharedClusterMap) { - computePointerWithAlignment(mem, mClusterStateExt, mNMaxClusters); + computePointerWithAlignment(mem, mClusterStateExt, mNClusters); } else { mClusterStateExt = nullptr; } @@ -497,33 +502,33 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() void GPUTPCGMMerger::SetMaxData(const GPUTrackingInOutPointers& io) { mNTotalSectorTracks = 0; - mNClusters = 0; + mNSectorHits = 0; mNMaxSingleSectorTracks = 0; for (int32_t iSector = 0; iSector < NSECTORS; iSector++) { uint32_t ntrk = *mRec->GetConstantMem().tpcTrackers[iSector].NTracks(); mNTotalSectorTracks += ntrk; - mNClusters += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); + mNSectorHits += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); if (mNMaxSingleSectorTracks < ntrk) { mNMaxSingleSectorTracks = ntrk; } } - mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNClusters); + mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNSectorHits); if (CAMath::Abs(Param().polynomialField.GetNominalBz()) < (gpu_common_constants::kZeroFieldCut * gpu_common_constants::kCLight)) { mNMaxTracks = mRec->MemoryScalers()->getValue(mNTotalSectorTracks, mNTotalSectorTracks); // 0 magnetic field } else { mNMaxTracks = mRec->MemoryScalers()->NTPCMergedTracks(mNTotalSectorTracks); } if (io.clustersNative) { - mNMaxClusters = io.clustersNative->nClustersTotal; + mNClusters = io.clustersNative->nClustersTotal; } else if (mRec->GetRecoSteps() & gpudatatypes::RecoStep::TPCSectorTracking) { - mNMaxClusters = 0; + mNClusters = 0; for (int32_t i = 0; i < NSECTORS; i++) { - mNMaxClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); + mNClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); } } else { - mNMaxClusters = mNClusters; + mNClusters = mNSectorHits; } - mNMaxLooperMatches = mNMaxClusters / 4; // We have that much scratch memory anyway + mNMaxLooperMatches = mNClusters / 4; // We have that much scratch memory anyway } int32_t GPUTPCGMMerger::CheckSectors() @@ -1657,7 +1662,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; } nHits += nTrackHits; } @@ -1838,11 +1843,13 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { for (uint32_t j = 0; j < trk.NClusters(); j++) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; + if (!Param().rec.tpc.rebuildTrackInFit) { + uint32_t weight = attachAttached | attachGood; + if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { + weight |= attachProtect; + } + CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); } - CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } if (!trk.CCE() && !trk.MergedLooper()) { @@ -1873,8 +1880,8 @@ GPUd() void GPUTPCGMMerger::PrepareForFit2(int32_t nBlocks, int32_t nThreads, in mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; } } - if (mClusterStateExt) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nBlocks * nThreads) { + if (!Param().rec.tpc.rebuildTrackInFit && mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); if (mSharedCount[i] > 1) { state |= GPUTPCGMMergedTrackHit::flagShared; @@ -1925,7 +1932,7 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nThreads * nBlocks) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; } @@ -2120,3 +2127,172 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif*/ } + +GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + mSharedCount2[i] = 0; + } + if (iThread == 0 && iBlock == 0) { + mMemory->nMergedTrackClusters = 0; + } + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } + } + } +} + +GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].id >= 2) { + const int32_t nCl = trk.NClusters(); + float weightf = nCl * (nCl * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + candidates[iteration].weight = weight; + CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + } else { + candidates[0].best = -1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { + candidates[0].best = iteration + 1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + int32_t nIter = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + trk.SetNClusters(0); + continue; + } + uint32_t nHits = 0; + uint8_t sharedRowCandidates[GPUTPCGeometry::NROWS]; + uint8_t nSharedRowCandidates = 0; + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + candidates[0].best = nIter; + } + if (candidates[0].best > 0) { + nHits++; + if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { + mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations + } + } else if (candidates[0].best == 0 && candidates[0].id >= 2) { + sharedRowCandidates[nSharedRowCandidates++] = j; + } + } + + if (nSharedRowCandidates) { + const uint32_t nShared = CAMath::Min(nSharedRowCandidates, CAMath::Float2UIntRn(nHits * Param().rec.tpc.rebuildTrackMaxSharedFraction)); + if (nShared) { + const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + const auto& ca = candidates[a * n]; + const auto& cb = candidates[b * n]; + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + }); + for (uint32_t j = 0; j < nShared; j++) { + auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); + cj[0].best = 1; + mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + } + nHits += nShared; + } + } + + const uint32_t iOut = CAMath::AtomicAdd(&mMemory->nMergedTrackClusters, nHits); + if (iOut + nHits > mNMaxMergedTrackClusters) { + raiseError(GPUErrors::ERROR_MERGER_REBUILD_HIT_OVERFLOW, iOut, mNMaxMergedTrackClusters); + CAMath::AtomicExch(&mMemory->nMergedTrackClusters, mNMaxMergedTrackClusters); + trk.SetNClusters(0); + trk.SetOK(false); + continue; + } + + trk.SetNClusters(nHits); + trk.SetFirstClusterRef(iOut); + uint32_t written = 0; + const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best > 0) { + auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; + if (candidates[0].best == 128) { + outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; + written++; + continue; + } + const auto& best = candidates[candidates[0].best - 1]; + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + written++; + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nBlocks * nThreads) { + if (mSharedCount2[mClusters[i].num]) { + mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; // TODO: What is the best criterion to set the shared flag? Perhaps already when iter0 best hit would be shared? + } + } + if (mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); + if (mSharedCount2[i]) { + state |= GPUTPCGMMergedTrackHit::flagShared; + } + mClusterStateExt[i] = state; + } + } +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 2afca0512783d..30c76aa2af810 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -83,6 +83,12 @@ class GPUTPCGMMerger : public GPUProcessor uint8_t row; uint8_t sector; float error; + int32_t weight; + int32_t best; + }; + + struct trackRebuildHelper { + bool reverse; }; struct tmpSort { @@ -119,14 +125,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdxAlt() const { return mMergedTracksdEdxAlt; } GPUhdi() GPUdEdxInfo* MergedTracksdEdxAlt() { return mMergedTracksdEdxAlt; } + GPUhdi() uint32_t NSectorHits() const { return mNSectorHits; } GPUhdi() uint32_t NClusters() const { return mNClusters; } - GPUhdi() uint32_t NMaxClusters() const { return mNMaxClusters; } GPUhdi() uint32_t NMaxTracks() const { return mNMaxTracks; } GPUhdi() uint32_t NMaxMergedTrackClusters() const { return mNMaxMergedTrackClusters; } GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } - GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } - GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } + GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } + GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -139,6 +146,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() tmpSort* TrackSortO2() { return mTrackSortO2; } GPUhdi() internal::MergeLooperParam* LooperCandidates() { return mLooperCandidates; } GPUhdi() GPUAtomic(uint32_t) * SharedCount() { return mSharedCount; } + GPUhdi() uint8_t* SharedCount2() { return mSharedCount2; } GPUhdi() gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) { return mBorderRange[i]; } GPUhdi() const gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) const { return mBorderRange[i]; } GPUhdi() GPUTPCGMBorderTrack* BorderTracks(int32_t i) { return mBorder[i]; } @@ -196,6 +204,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -261,7 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t mNMaxTracks = 0; // maximum number of output tracks uint32_t mNMaxSingleSectorTracks = 0; // max N tracks in one sector uint32_t mNMaxMergedTrackClusters = 0; // max number of clusters in output tracks (double-counting shared clusters) - uint32_t mNMaxClusters = 0; // max total unique clusters (in event) + uint32_t mNClusters = 0; // max total unique clusters (in event) uint32_t mNMaxLooperMatches = 0; // Maximum number of candidate pairs for looper matching uint16_t mMemoryResMemory = (uint16_t)-1; @@ -272,9 +285,12 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; - int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks trackCluster* mClusterCandidates = nullptr; + trackRebuildHelper* mTrackRebuildHelper = nullptr; + int32_t* mHitWeights = nullptr; + GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks @@ -295,6 +311,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t* mTrackSort = nullptr; tmpSort* mTrackSortO2 = nullptr; GPUAtomic(uint32_t) * mSharedCount = nullptr; // Must be uint32_t unfortunately for atomic support + uint8_t* mSharedCount2 = nullptr; GPUTPCGMBorderTrack* mBorderMemory = nullptr; // memory for border tracks GPUTPCGMBorderTrack* mBorder[2 * NSECTORS]; gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 9f6858df4d3f2..c9801162cef40 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -192,7 +192,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const out << " Track Clusters"; DumpTrackClusters(out); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (j++ % 10 == 0) { out << "\n Cluster attachment "; @@ -241,7 +241,7 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "Track Clusters\n"; DumpTrackClusters(out, true); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (++j % 10 == 0) { out << " Cluster attachment "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 22d8b159e9b74..d3ea60b4dc099 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -214,3 +214,34 @@ GPUdii() void GPUTPCGMMergerMergeLoopers::Thread<2>(int32_t nBlocks, int32_t nTh { merger.MergeLoopersMain(nBlocks, nThreads, iBlock, iThread); } + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.PrepareHitWeights(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ComputeHitWeights(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights1(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights2(nBlocks, nThreads, iBlock, iThread); +} + + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeightsShared(nBlocks, nThreads, iBlock, iThread); +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index b4f823aa11ed8..82605f47a5f27 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -180,6 +180,19 @@ class GPUTPCGMMergerMergeLoopers : public GPUTPCGMMergerGeneral GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; +class GPUTPCGMMergerHitWeights : public GPUTPCGMMergerGeneral +{ + public: + enum K { defaultKernel = 0, + prepare = 0, + compute = 1, + resolve1 = 2, + resolve2 = 3, + resolveShared = 4 }; + template + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t iteration); +}; + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c5484cc8f523c..42b65abeca828 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -136,26 +136,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { - while (true) { - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - } - const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (interHit.id == 1) { - nMissed++; - nMissed2++; - continue; - } - const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - if (interHit.id - 2 == clusters[ihit].num) { - clusterState = clusters[ihit].state; - } else { - clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -227,7 +208,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -303,13 +284,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } + CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -1042,9 +1024,13 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); - track.Param() = t; - track.Alpha() = Alpha; + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.Param().NDF() = 0; + } else { + track.Param() = t; + track.Alpha() = Alpha; + } if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index 7c9047202417b..aaf1bff882522 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -162,9 +162,12 @@ class GPUTPCTracker : public GPUProcessor GPUdi() static int32_t CalculateHitWeight(int32_t NHits, float chi2) { float weight = NHits * (NHits * 2 - 5) * 128 / chi2; // TODO: Add QPt to this formula - if (weight < 0.f || weight > 2e9f) { + if (weight < 0.f) { return 0; } + if (weight > 2e9f) { + return 2e9f; + } return ((int32_t)weight); } GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 1b2403af07733..38ca9adb3d242 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -996,7 +996,7 @@ int32_t main(int argc, char** argv) nEventsProcessed++; if (configStandalone.timeFrameTime) { - double nClusters = chainTracking->GetProcessors()->tpcMerger.NMaxClusters(); + double nClusters = chainTracking->GetProcessors()->tpcMerger.NClusters(); if (nClusters > 0) { const int32_t nOrbits = 32; const double colRate = 50000; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 138eb8a6339db..539840baaff7a 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -84,6 +84,11 @@ o2_gpu_add_kernel("GPUTPCGMMergerFinalize, step2" "GPUTP o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step0" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step1" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step2" "GPUTPCGMMergerGPU TPCMERGER" LB) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, prepare" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, compute" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve1" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve2" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolveShared" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) o2_gpu_add_kernel("GPUTPCGMO2Output, prepare" "= TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMO2Output, sort" "= TPCMERGER") o2_gpu_add_kernel("GPUTPCGMO2Output, output" "= TPCMERGER" LB) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 86aaea47b57f0..689f9a36fc2fe 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -1870,7 +1870,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } - uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NMaxClusters(); + uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NClusters(); mClusterCounts.nTotal += nCl; if (mQATasks & (taskClusterCounts | taskClusterRejection)) { for (uint32_t iSector = 0; iSector < GPUTPCGeometry::NSECTORS; iSector++) { From 927a7980c5c2644417329338131f416c76e2c1a7 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Feb 2026 20:39:07 +0100 Subject: [PATCH 08/60] GPU: Additional debug dumps --- .../Global/GPUChainTrackingMerger.cxx | 2 + GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 7f393446b2744..52cdcd5dee78f 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -243,7 +243,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 30c76aa2af810..d07a47d4fe981 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -222,6 +222,8 @@ class GPUTPCGMMerger : public GPUProcessor void DumpRefit(std::ostream& out) const; void DumpFinal(std::ostream& out) const; void DumpLoopers(std::ostream& out) const; + void DumpInterpolatedHits(std::ostream& out) const; + void DumpRebuiltTracks(std::ostream& out) const; void DumpTrackParam(std::ostream& out) const; void DumpTrackClusters(std::ostream& out, bool non0StateOnly = false, bool noNDF0 = false) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index c9801162cef40..d22fd2e05e0a0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -150,6 +150,15 @@ void GPUTPCGMMerger::DumpTrackParam(std::ostream& out) const out << std::setprecision(ss); } +void GPUTPCGMMerger::DumpRebuiltTracks(std::ostream& out) const +{ + out << "\nTPC Merger Rebuilt Tracks\n"; + out << " Cluster Attachment\n"; + DumpTrackClusters(out, false, true); + out << " Track Params\n"; + DumpTrackParam(out); +} + void GPUTPCGMMerger::DumpMergeCE(std::ostream& out) const { DumpTrackLinks(out, true, " for CE merging"); @@ -255,6 +264,34 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "\n"; } +void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const +{ + out << "\nTPC Merger Interpolated Hits\n"; + for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) { + const auto& trk = mMergedTracks[i]; + if (trk.OK() && trk.GetParam().GetNDF() >= 0 && trk.NClusters()) { + out << "Track " << i << ":"; + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].id) { + out << " Row " << j << ": "; + if (candidates[0].best) { + out << " Best " << candidates[0].best << " - "; + } + for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { + if (candidates[k].id) { + out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + } + } + out << "; "; + } + } + out << "\n"; + } + } + out << "\n"; +} + template inline void GPUTPCGMMerger::MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, uint8_t mergeMode, float weight, float frac) const { From 976b10f12486423706f0255772b83d5dd8e23272 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 6 Oct 2025 22:09:55 +0200 Subject: [PATCH 09/60] GPU TPC: Split TPC Merger Fit() into multiple functions --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 652 +++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 46 +- 4 files changed, 377 insertions(+), 330 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index d9b139b73ab7e..ef5abc7e49a31 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -571,7 +571,7 @@ GPUd() int32_t GPUTPCGMMerger::RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack trk.QPt() = inTrack->Param().GetQPt(); trk.TOffset() = Param().par.continuousTracking ? GetConstantMem()->calibObjects.fastTransform->convZOffsetToVertexTime(sector, inTrack->Param().GetZOffset(), Param().continuousMaxTimeBin) : 0; const auto tmp = sectorTrack.ClusterTN() > sectorTrack.ClusterT0() ? std::array{sectorTrack.ClusterTN(), sectorTrack.ClusterT0()} : std::array{sectorTrack.ClusterT0(), sectorTrack.ClusterTN()}; - trk.ShiftZ(this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead + trk.ShiftZ(*this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead sectorTrack.SetX2(0.f); for (int32_t way = 0; way < 2; way++) { if (way) { @@ -1858,7 +1858,7 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in const auto& GPUrestrict() cls = GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[cl0.num].getTime(), zn = cls[cln.num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(cln.row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(cl0.row)}; - trk.Param().ShiftZ(this, cl0.sector, tmp[0], tmp[1], tmp[2]); + trk.Param().ShiftZ(*this, cl0.sector, tmp[0], tmp[1], tmp[2]); updTrk = &trk; while (updTrk->PrevSegment() >= 0) { auto next = &mMergedTracks[updTrk->PrevSegment()]; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index d3ea60b4dc099..083cdf76699a0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -23,7 +23,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); }); } @@ -31,7 +31,7 @@ template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { - GPUTPCGMTrackParam::PropagateLooper(&merger, i); + GPUTPCGMTrackParam::PropagateLooper(merger, i); }); } @@ -239,7 +239,6 @@ GPUdii() void GPUTPCGMMergerHitWeights::Thread GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 42b65abeca828..fdfddec220604 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,13 +47,13 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { - constexpr float kDeg2Rad = M_PI / 180.f; constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); - const GPUParam& GPUrestrict() param = merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); + GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; GPUTPCGMPropagator prop; @@ -86,13 +86,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); - prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -128,13 +127,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); + // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) + // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; @@ -169,253 +168,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); - const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); - GPUglobalref() const cahit2* hits = tracker.HitData(row); - GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; - const float y0 = row.Grid().YMin(); - const float stepY = row.HstepY(); - const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway - const float stepZ = row.HstepZ(); - int32_t bin, ny, nz; - - float err2Y, err2Z; - param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - - float uncorrectedZ; - merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); - - int32_t nCandidates = 0; - if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { - const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); - const float tubeY = CAMath::Sqrt(sy2); - const float tubeZ = CAMath::Sqrt(sz2); - row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); - - const int32_t nBinsY = row.Grid().Ny(); - const int32_t idOffset = tracker.Data().ClusterIdOffset(); - const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - for (int32_t k = 0; k <= nz; k++) { - const int32_t mybin = bin + k * nBinsY; - const uint32_t hitFst = firsthit[mybin]; - const uint32_t hitLst = firsthit[mybin + ny + 1]; - for (uint32_t ih = hitFst; ih < hitLst; ih++) { - const cahit2 hh = hits[ih]; - const float y = y0 + hh.x * stepY; - const float z = z0 + hh.y * stepZ; - const float dy = y - uncorrectedY; - const float dz = z - uncorrectedZ; - - if (dy * dy < sy2 && dz * dz < sz2) { - float err2YA, err2ZA; - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; - const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - const float time = cl.getTime(); - const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); - const float invCharge = 1.f / cl.qMax; - float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); - invAvgCharge *= invAvgCharge; - - prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); - const float Jw0 = 1.f / (ImC0 + err2YA); - const float Jw2 = 1.f / (ImC2 + err2ZA); - const float chi2Y = Jw0 * dy * dy; - const float chi2Z = Jw2 * dz * dz; - bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; - if (ok) { - int32_t insert = nCandidates; - for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { - insert = c; - break; - } - } - if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { - for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; - nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); - } - } - } - } - } - CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); - } - if (nCandidates == 0) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; - } - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); + if (retValHit == 1) { break; - } - if (retValProp || sinPhiErr) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - NTolerated++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + } else if (retValHit == -1) { continue; } - CADEBUG(printf("\n")); - - int32_t retValUpd = 0, retValInt = 0; - float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); - if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - - float err2Y, err2Z; - const float time = merger->GetConstantMem()->ioPtrs.clustersNative ? merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; - const float invSqrtCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - const float invCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; - invAvgCharge *= invAvgCharge; - prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); - } - } - - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; - } else { - retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); - } - GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); - }); - } - // clang-format off - CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); - CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); - // clang-format on - - ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! - if (!retValUpd && !retValInt) // track is updated - { - lastUpdateX = mX; - covYYUpd = mC[0]; - nMissed = nMissed2 = 0; - UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters - bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; - if (acc || accAlt) { - float qtot = 0, qmax = 0, pad = 0, relTime = 0; - const int32_t clusterCount = (ihit - ihitMergeFirst) * wayDirection + 1; - for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { - const ClusterNative& cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num]; - qtot += cl.getQtot(); - qmax = CAMath::Max(qmax, cl.qMax); - pad += cl.getPad(); - relTime += cl.getTime(); - } - qtot /= clusterCount; // TODO: Weighted Average - pad /= clusterCount; - relTime /= clusterCount; - relTime = relTime - CAMath::Round(relTime); - if (acc) { - dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - if (accAlt) { - dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); - } - } - } - } - } - } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track - if (retValInt || allowChangeClusters) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); - } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - } - if (!retValInt) { - nMissed++; - nMissed2++; - } - } else { - break; // bad chi2 for the whole track, stop the fit - } } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); @@ -449,9 +209,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } if (param.par.dodEdx && param.dodEdxEnabled) { - dEdx.computedEdx(merger->MergedTracksdEdx()[iTrk], param); + dEdx.computedEdx(merger.MergedTracksdEdx()[iTrk], param); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.computedEdx(merger->MergedTracksdEdxAlt()[iTrk], param); + dEdxAlt.computedEdx(merger.MergedTracksdEdxAlt()[iTrk], param); } } Alpha = prop.GetAlpha(); @@ -461,6 +221,277 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ return true; } +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const auto& cluster = clusters[ihit]; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + // clang-format on + if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + { + CADEBUG(printf("REROTATE\n")); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { + retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + } + } + if (retValProp == 0) { + lastRow = cluster.row; + lastSector = cluster.sector; + } + // clang-format off + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } + + static constexpr float kDeg2Rad = M_PI / 180.f; + const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + return 1; + } + if (retValProp || sinPhiErr) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + NTolerated++; + CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + return -1; + } + CADEBUG(printf("\n")); + + int32_t retValUpd = 0, retValInt = 0; + float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); + if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + } else { + int8_t rejectChi2 = 0; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; + } + } + + float err2Y, err2Z; + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); + + if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = GPUTPCGMPropagator::rejectDirect; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + } + } + + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; + } else { + retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); + } + GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { + merger.DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + }); + } + // clang-format off + CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); + CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); + // clang-format on + + ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! + if (!retValUpd && !retValInt) // track is updated + { + lastUpdateX = mX; + covYYUpd = mC[0]; + nMissed = nMissed2 = 0; + UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + N++; + ihitStart = ihit; + float dy = mP[0] - prop.Model().Y(); + float dz = mP[1] - prop.Model().Z(); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } + FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track + if (retValInt || allowChangeClusters) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); + } else if (finalFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + if (!retValInt) { + nMissed++; + nMissed2++; + } + } else { + return 1; // bad chi2 for the whole track, stop the fit + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() rowData = tracker.Row(row); + GPUglobalref() const cahit2* hits = tracker.HitData(rowData); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); + float uncorrectedY = -1e6f, uncorrectedZ; + if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; + const float y0 = rowData.Grid().YMin(); + const float stepY = rowData.HstepY(); + const float z0 = rowData.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = rowData.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = rowData.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + } + if (nCandidates == 0) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + return uncorrectedY; +} + +GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +{ + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters + bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; + if (acc || accAlt) { + float qtot = 0, qmax = 0, pad = 0, relTime = 0; + const int32_t clusterCount = CAMath::Abs(ihit - ihitMergeFirst) + 1; + for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { + const ClusterNative& cl = clustersArray[clusters[iTmp].num]; + qtot += cl.getQtot(); + qmax = CAMath::Max(qmax, cl.qMax); + pad += cl.getPad(); + relTime += cl.getTime(); + } + qtot /= clusterCount; // TODO: Weighted Average + pad /= clusterCount; + relTime /= clusterCount; + relTime = relTime - CAMath::Round(relTime); + const auto& cluster = clusters[ihit]; + if (acc) { + dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + if (accAlt) { + dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); + } + } + } + } + } +} + GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { constexpr float kDeg2Rad = M_PI / 180.f; @@ -518,11 +549,11 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger* GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger->Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge maxDistY = (maxDistY + mC[0]) * 20.f; maxDistZ = (maxDistZ + mC[2]) * 20.f; int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) @@ -537,10 +568,10 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t xx = yy = zz = 0.f; clusterState = 0; while (true) { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; float clamp = cl.getQtot(); float clx, cly, clz; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); float dy = cly - projY; float dz = clz - projZ; if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { @@ -572,25 +603,25 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); if (prop.GetPropagatedYZ(X, Y, Z)) { Y = mP[0]; Z = mP[1]; } - return AttachClusters(Merger, sector, iRow, iTrack, goodLeg, Y, Z); + return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { - const auto& param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); if (param.rec.tpc.disableRefitAttachment & 1) { return -1e6f; } - const GPUTPCTracker& GPUrestrict() tracker = *(Merger->GetConstantMem()->tpcTrackers + sector); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); @@ -598,7 +629,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return -1e6f; } - const float zOffset = param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -606,7 +637,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric int32_t bin, ny, nz; float uncorrectedY, uncorrectedZ; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { return uncorrectedY; } @@ -618,7 +649,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - float time = Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform + float time = merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform if (iRow < param.rec.tpc.tubeExtraProtectMinRow || pad < param.rec.tpc.tubeExtraProtectEdgePads || pad >= (uint32_t)(GPUTPCGeometry::NPads(iRow) - param.rec.tpc.tubeExtraProtectEdgePads) || param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { @@ -635,8 +666,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = Merger->TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; - GPUAtomic(uint32_t)* const weights = Merger->ClusterAttachment(); + uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; } @@ -670,10 +701,10 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) +GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { constexpr float kSectAngle = 2 * M_PI / 18.f; - if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { + if (merger.Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } if (CAMath::Abs(lastRow - toRow) < 2) { @@ -694,14 +725,14 @@ GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GP } if (dodEdx && iRow + step == toRow) { float yUncorrected, zUncorrected; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (Merger->GetConstantMem()->calibObjects.dEdxCalibContainer && Merger->GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { dodEdx = false; } } CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(Merger, sector, iRow, iTrack, goodLeg, prop); + AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); } return dodEdx; } @@ -719,7 +750,7 @@ GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterPar outerParam->alpha = alpha; } -GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) +GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) { if (iRow == 0 || iRow == GPUTPCGeometry::NROWS - 1) { return; @@ -734,10 +765,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* return; } - uint32_t nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1u); - if (nLoopData >= Merger->NMaxTracks()) { - Merger->raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, Merger->NMaxTracks()); - CAMath::AtomicExch(&Merger->Memory()->nLoopData, Merger->NMaxTracks()); + uint32_t nLoopData = CAMath::AtomicAdd(&merger.Memory()->nLoopData, 1u); + if (nLoopData >= merger.NMaxTracks()) { + merger.raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, merger.NMaxTracks()); + CAMath::AtomicExch(&merger.Memory()->nLoopData, merger.NMaxTracks()); return; } GPUTPCGMLoopData data; @@ -747,36 +778,36 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* data.sector = sector; data.row = iRow; data.outwards = outwards; - Merger->LoopData()[nLoopData] = data; + merger.LoopData()[nLoopData] = data; } -GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx) +GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { GPUTPCGMPropagator prop; prop.SetMaterialTPC(); - prop.SetPolynomialField(&Merger->Param().polynomialField); + prop.SetPolynomialField(&merger.Param().polynomialField); prop.SetMaxSinPhi(constants::MAX_SIN_PHI); - prop.SetMatLUT(Merger->Param().rec.useMatLUT ? Merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT(merger.Param().rec.useMatLUT ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetSeedingErrors(false); prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = Merger->LoopData()[loopIdx]; + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (false) { - data.param.AttachClustersLooper(Merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } else { - data.param.AttachClustersLooperFollow(Merger, prop, data.sector, data.track, data.outwards); + data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) { float toX = mX; - bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; + bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; constexpr float kSectAngle = 2 * M_PI / 18.f; - const GPUParam& GPUrestrict() param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; float lrFactor = right ^ !up ? 1.f : -1.f; @@ -813,7 +844,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(Merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -839,7 +870,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -889,31 +920,32 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(Merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, int32_t N) { if (N == 0) { N = 1; } - const auto& GPUrestrict() cls = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear; + const auto& GPUrestrict() cls = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[clusters[0].num].getTime(), zn = cls[clusters[N - 1].num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(clusters[N - 1].row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(clusters[0].row)}; return ShiftZ(merger, clusters[0].sector, tmp[0], tmp[1], tmp[2]); } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) { - if (!merger->Param().par.continuousTracking) { + const GPUParam& GPUrestrict() param = merger.Param(); + if (!param.par.continuousTracking) { return 0.f; } float deltaZ = 0.f; bool beamlineReached = false; - const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * merger->Param().polynomialField.GetNominalBz())); + const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * param.polynomialField.GetNominalBz())); if (r1 < 0.01501) { // 100 MeV @ 0.5T ~ 0.66m cutof const float dist2 = mX * mX + mP[0] * mP[0]; const float dist1r2 = dist2 * r1 * r1; @@ -940,16 +972,16 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg } if (!beamlineReached) { - float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? merger->Param().rec.tpc.defaultZOffsetOverR : -merger->Param().rec.tpc.defaultZOffsetOverR) * clx; + float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? param.rec.tpc.defaultZOffsetOverR : -param.rec.tpc.defaultZOffsetOverR) * clx; float basez; - merger->GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } { - float deltaT = merger->GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + float deltaT = merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); mTOffset += deltaT; - const float maxT = cltmin - merger->GetConstantMem()->calibObjects.fastTransform->getT0(); - const float minT = cltmax - merger->GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); deltaT = 0.f; if (mTOffset < minT) { @@ -959,7 +991,7 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg deltaT = maxT - mTOffset; } if (deltaT != 0.f) { - deltaZ += merger->GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; } @@ -1001,7 +1033,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -1015,27 +1047,27 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); - CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } - CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); + CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? - if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { track.Param() = t; track.Alpha() = Alpha; } - if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } - // if (track.OK()) merger->DebugRefitMergedTrack(track); + // if (track.OK()) merger.DebugRefitMergedTrack(track); } GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index b287e5a717edf..a4ca605e5d547 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -27,8 +27,14 @@ class AliExternalTrackParam; +namespace o2::tpc +{ +struct ClusterNative; +}; + namespace o2::gpu { +class GPUTPCTracker; class GPUTPCGMMerger; class GPUTPCGMBorderTrack; struct GPUParam; @@ -36,6 +42,13 @@ class GPUTPCGMPhysicalTrackModel; class GPUTPCGMPolynomialField; class GPUTPCGMMergedTrack; class GPUTPCGMPropagator; +class GPUdEdx; +struct GPUTPCGMMergedTrackHit; + +namespace gputpcgmmergertypes +{ +struct InterpolationErrorHit; +} // namespace gputpcgmmergertypes /** * @class GPUTPCGMTrackParam @@ -141,23 +154,26 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); - - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + + GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); - GPUd() static void PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx); + GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); - GPUd() void AddCovDiagErrors(const float* GPUrestrict() errors2); - GPUd() void AddCovDiagErrorsWithCorrelations(const float* GPUrestrict() errors2); + GPUd() void AddCovDiagErrors(const float* errors2); + GPUd() void AddCovDiagErrorsWithCorrelations(const float* errors2); GPUdi() void MarkClusters(GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihitFirst, int32_t ihitLast, int32_t wayDirection, uint8_t state) { @@ -183,8 +199,8 @@ class GPUTPCGMTrackParam } GPUd() void Rotate(float alpha); - GPUd() float ShiftZ(const GPUTPCGMMerger* merger, uint32_t sector, float cltmax, float cltmin, float clx); - GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N); + GPUd() float ShiftZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float clx); + GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From 004831c678c8140d279c675d446b4ced70770bb3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:01 +0200 Subject: [PATCH 10/60] GPU: Add looperFollowMode option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index a7b96b692cb35..4f70ebca451f3 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -144,6 +144,7 @@ AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fractio AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index fdfddec220604..29072163ec067 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -794,10 +794,10 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); - if (false) { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); - } else { + if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); + } else { + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } } From 42ffc950df10bef1a5a9ba81757e1c4c75833fb5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:20 +0200 Subject: [PATCH 11/60] GPU: simplifaction, refactoring and some minor improvements --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 145 ++++++------- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 196 ++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 13 +- 5 files changed, 174 insertions(+), 194 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 4f70ebca451f3..34322048511ae 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -138,7 +138,7 @@ AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger ( AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") +AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 974a0b3ea2074..49e0773abab1d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -608,7 +608,7 @@ GPUd() float GPUTPCGMPropagator::PredictChi2(float posY, float posZ, float err2Y } } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) { float err2Y, err2Z; GetErr2(err2Y, err2Z, param, posZ, iRow, clusterState, sector, time, avgInvCharge, invCharge); @@ -616,7 +616,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, iRow, param, clusterState, rejectChi2, refit, err2Y, err2Z); } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z) { if (mT->NDF() == -5) { // first measurement: no need to filter, as the result is known in advance. just set it. mT->ResetCovariance(); @@ -635,80 +635,83 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return 0; } - return Update(posY, posZ, clusterState, rejectChi2 == rejectDirect, err2Y, err2Z, ¶m); + return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); - if (rejectChi2 == rejectInterFill) { - inter->posY = mP[0]; - inter->posZ = mP[1]; - if (mT->NDF() <= 0) { - inter->errorY = inter->errorZ = 100.f; - } else { - inter->errorY = mC[0]; - inter->errorZ = mC[2]; - } - } else if (rejectChi2 == rejectInterReject) { - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } + inter->posY = mP[0]; + inter->posZ = mP[1]; + if (mT->NDF() <= 0) { + inter->errorY = inter->errorZ = 100.f; + } else { + inter->errorY = mC[0]; + inter->errorZ = mC[2]; + } +} + +GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +{ + float* GPUrestrict() mC = mT->Cov(); + float* GPUrestrict() mP = mT->Par(); + float chi2Y, chi2Z; + if (mT->NDF() <= 0) { + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (mFitInProjections) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return updateErrorClusterRejectedInInterpolation; } return 0; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index c3f7aac84a1a5..bf67d027f6b05 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -57,11 +57,6 @@ class GPUTPCGMPropagator updateErrorClusterRejectedInUpdate = 4, updateErrorClusterRejectedEdge = 5 }; - enum RejectChi2Mode { - rejectDirect = 1, - rejectInterFill = 2, - rejectInterReject = 3 - }; GPUdDefault() GPUTPCGMPropagator() = default; @@ -104,10 +99,11 @@ class GPUTPCGMPropagator GPUd() int32_t PropagateToXAlphaBz(float posX, float posAlpha, bool inFlyDirection); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); + GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 29072163ec067..1ee9cc8115954 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); + constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -104,11 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayDirection = (iWay & 1) ? -1 : 1; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); - if (crossCE) { - lastSector = clusters[ihit].sector; - } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { @@ -130,11 +125,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } - // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); - // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; @@ -151,33 +142,64 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const auto& cluster = clusters[ihit]; - - // clang-format off - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - // clang-format on - if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; - dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); - if (dodEdx) { - dEdx.fillSubThreshold(lastRow - wayDirection); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(lastRow - wayDirection); + if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (dodEdx || doAttach || doIntertpolate) { + int32_t step = cluster.row > lastRow ? 1 : -1; + float tmpX, tmpY, tmpZ; + for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { + if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { + break; + } + if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + break; + } + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { + dEdx.fillSubThreshold(iRow); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + dEdxAlt.fillSubThreshold(iRow); + } + } + } + } + if (doAttach) { + AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); } } } } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); - if (retValHit == 1) { + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + if (retValHit == 0) { + ihitStart = ihit; + N++; + covYYUpd = mC[0]; + } else if (retValHit == 1) { break; - } else if (retValHit == -1) { + } else if (retValHit == 2) { + NTolerated++; continue; } + + lastRow = cluster.row; + lastSector = cluster.sector; } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -221,7 +243,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -238,26 +260,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } // clang-format off CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -269,33 +275,33 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; + return 1; // bad chi2 for the whole track, stop the fit } if (retValProp || sinPhiErr) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - NTolerated++; CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); - return -1; + return 2; // Propagate failed or high incl angle } CADEBUG(printf("\n")); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; @@ -305,12 +311,23 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + bool rejectChi2 = false; + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 3) { + prop.InterpolateFill(yy, zz, &inter); + } else if (iWay == nWays - 2) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else if (!param.rec.tpc.rebuildTrackInFit) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } } + } else { + rejectChi2 = GetNDF() > 0 && allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters @@ -331,11 +348,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - covYYUpd = mC[0]; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; float dy = mP[0] - prop.Model().Y(); float dz = mP[1] - prop.Model().Z(); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { @@ -343,6 +357,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.SetTrack(this, prop.GetAlpha()); } FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -353,10 +368,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger nMissed++; nMissed2++; } + return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit } - return 0; } GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) @@ -392,6 +407,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); int32_t nCandidates = 0; + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + nCandidates++; + } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? @@ -701,42 +719,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) -{ - constexpr float kSectAngle = 2 * M_PI / 18.f; - if (merger.Param().rec.tpc.disableRefitAttachment & 2) { - return dodEdx; - } - if (CAMath::Abs(lastRow - toRow) < 2) { - return dodEdx; - } - int32_t step = toRow > lastRow ? 1 : -1; - float xx = mX - GPUTPCGeometry::Row2X(lastRow); - for (int32_t iRow = lastRow + step; iRow != toRow; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi) { - return dodEdx; - } - if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - return dodEdx; - } - int32_t err = prop.PropagateToXAlpha(xx + GPUTPCGeometry::Row2X(iRow), prop.GetAlpha(), inFlyDirection); - if (err) { - return dodEdx; - } - if (dodEdx && iRow + step == toRow) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { - dodEdx = false; - } - } - CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); - } - return dodEdx; -} - GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) { CADEBUG(printf("\t%21sStorO Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f, SP %5.2f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f\n", "", alpha, mX, mP[0], mP[1], mP[4], mP[2], sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]))); @@ -801,7 +783,7 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -870,7 +852,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -880,7 +862,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float SinPhi = CAMath::Sqrt(1 - mP[2] * mP[2]) * (mP[2] > 0 ? -1 : 1); float b = prop.GetBz(prop.GetAlpha(), mX, mP[0], mP[1]); - float dx = outwards ? 1.f : -1.f; + float dx = up ? 1.f : -1.f; const float myRowX = GPUTPCGeometry::Row2X(iRow); // printf("\nAttachMirror sector %d row %d outwards %d\n", (int)sector, (int)iRow, (int)outwards); // printf("X %f Y %f Z %f SinPhi %f -->\n", mX, mP[0], mP[1], mP[2]); @@ -914,7 +896,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr // printf("count %d: At X %f Y %f Z %f SinPhi %f\n", maxTries, mP[2] > 0 ? -Y : Y, mP[2] > 0 ? X : -X, Z, SinPhi); float paramX = mP[2] > 0 ? -Y : Y; - int32_t step = outwards ? 1 : -1; + int32_t step = up ? 1 : -1; int32_t found = 0; for (int32_t j = iRow; j >= 0 && j < (int32_t)GPUTPCGeometry::NROWS && found < 3; j += step) { float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a4ca605e5d547..d738f09a91c15 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,19 +156,18 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 6f0cecb68927b6b42815278ee397c7f3f0a1cbd1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 13:34:27 +0100 Subject: [PATCH 12/60] GPU TPC: Refactor cluster merging --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 98 +++++++++++-------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 1ee9cc8115954..86b179522bfcc 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -121,26 +121,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - } CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; } - if (param.rec.tpc.rejectIFCLowRadiusCluster) { - const float r2 = xx * xx + yy * yy; - const float rmax = (83.5f + param.rec.tpc.sysClusErrorMinDist); - if (r2 < rmax * rmax) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - continue; - } - } - const auto& cluster = clusters[ihit]; if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; @@ -569,55 +556,86 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { + const int32_t ihitFirst = ihit; + { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + } if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { - float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge - maxDistY = (maxDistY + mC[0]) * 20.f; - maxDistZ = (maxDistZ + mC[2]) * 20.f; - int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) + float maxDistY2, maxDistZ2; + bool noReject = false; // Cannot reject if simple estimation of y/z fails (extremely unlike case) if (CAMath::Abs(clAlpha - prop.GetAlpha()) > 1.e-4f) { noReject = prop.RotateToAlpha(clAlpha); } float projY = 0, projZ = 0; - if (noReject == 0) { + if (!noReject) { noReject |= prop.GetPropagatedYZ(xx, projY, projZ); } - float count = 0.f; - xx = yy = zz = 0.f; - clusterState = 0; - while (true) { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - float clamp = cl.getQtot(); - float clx, cly, clz; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); - float dy = cly - projY; - float dz = clz - projZ; - if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ), mP[0], projY, cly, mP[1], projZ, clz)); + prop.GetErr2(maxDistY2, maxDistZ2, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; + maxDistY2 = (maxDistY2 + mC[0]) * kFactor; + maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; + auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + float dy = y - projY; + float dz = z - projZ; + if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { + CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (rejectChi2) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; + clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } + return false; } else { - CADEBUG(printf("\t\tMerging hit row %d X %f Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ihit].row, clx, cly, clz, dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ))); - xx += clx * clamp; // TODO: Weight in pad/time instead of XYZ + CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + return true; + } + }; + const float tmpX = xx; + float count; + if (chkFunction(ihit, yy, zz)) { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.getQtot(); + xx *= clamp; + yy *= clamp; + zz *= clamp; + clusterState = clusters[ihit].state; + count = clamp; + } else { + xx = yy = zz = count = 0.f; + clusterState = 0; + } + do { + ihit += wayDirection; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.getQtot(); + float clx, cly, clz; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + if (chkFunction(ihit, cly, clz)) { + xx += clx * clamp; yy += cly * clamp; zz += clz * clamp; clusterState |= clusters[ihit].state; count += clamp; } - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - ihit += wayDirection; - } + } while (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector); if (count < 0.1f) { CADEBUG(printf("\t\tNo matching cluster in double-row, skipping\n")); + xx = tmpX; return -1; } xx /= count; yy /= count; zz /= count; } + if (merger.Param().rec.tpc.rejectIFCLowRadiusCluster) { + const float r2 = xx * xx + yy * yy; + const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); + if (r2 < rmax2) { + if (rejectChi2) { + MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + return -1; + } + } return 0; } From 1eddd603f6d9310566e6d8129ecef67adbedf125 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 24 Oct 2025 11:01:17 +0200 Subject: [PATCH 13/60] GPU: Change indexing of interpolated hit temporary data --- GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 5 + GPU/GPUTracking/Global/GPUErrorCodes.h | 35 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 3 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 173 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 8 +- 6 files changed, 130 insertions(+), 99 deletions(-) diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index ed334aef3a75f..32fbe83357d6b 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -17,6 +17,9 @@ #include "GPUCommonDef.h" #include "DataFormatsTPC/Constants.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif namespace o2::gpu { @@ -174,6 +177,8 @@ class GPUTPCGeometry const float v = (sector >= NSECTORS / 2) ? -z : z; return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } + + GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index 2080548c7775c..b3894cf267c17 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -35,22 +35,23 @@ GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER -GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped -GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded -GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete -GPUCA_ERROR_CODE(29, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset -GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word -GPUCA_ERROR_CODE(31, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode -GPUCA_ERROR_CODE(32, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version -GPUCA_ERROR_CODE(33, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder -GPUCA_ERROR_CODE(33, MAX_GPUCA_ERROR_NUMBER) +GPUCA_ERROR_CODE(19, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER +GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped +GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded +GPUCA_ERROR_CODE(29, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete +GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset +GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word +GPUCA_ERROR_CODE(32, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode +GPUCA_ERROR_CODE(33, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version +GPUCA_ERROR_CODE(34, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder +GPUCA_ERROR_CODE(34, MAX_GPUCA_ERROR_NUMBER) // #define GPUCA_CHECK_TPCZS_CORRUPTION diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 89b553aba9d1c..d8b14ecc61182 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -38,7 +38,8 @@ struct InterpolationErrorHit { }; struct InterpolationErrors { - InterpolationErrorHit hit[constants::MERGER_MAX_TRACK_CLUSTERS]; + static constexpr size_t size = constants::MERGER_MAX_TRACK_CLUSTERS; + InterpolationErrorHit hit[size]; }; struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 86b179522bfcc..612a99de4d2d6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,6 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -62,7 +61,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (int32_t i = 0; i < N; i++) { + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } } @@ -70,6 +69,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; int32_t ihitStart = 0; + int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -94,86 +94,81 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); - CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); + CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastRow = 255; - uint8_t lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { - NTolerated++; - } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { + clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + } + continue; } - continue; } const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; + interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { - nMissed++; - nMissed2++; - continue; - } + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); const auto& cluster = clusters[ihit]; - if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + uint8_t dEdxSubThresholdRow = 255; + bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); - if (dodEdx || doAttach || doIntertpolate) { - int32_t step = cluster.row > lastRow ? 1 : -1; - float tmpX, tmpY, tmpZ; - for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - break; - } + if (dodEdx || doAttach || doInterpolate) { + int32_t step = cluster.row > lastPropagateRow ? 1 : -1; + for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - break; - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (dodEdx) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); - if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { - dEdx.fillSubThreshold(iRow); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(iRow); - } - } - } - } - if (doAttach) { - AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); - } + FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } + interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + } + lastPropagateRow = cluster.row; + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + if (doInterpolate) { + FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); + } + continue; } CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + if ((uint32_t)interpolationIndex >= interpolation.size) { + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); + break; + } + auto& inter = interpolation.hit[interpolationIndex]; + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); if (retValHit == 0) { + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; + interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; } else if (retValHit == 1) { @@ -183,12 +178,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - lastRow = cluster.row; - lastSector = cluster.sector; + lastUpdateRow = cluster.row; + lastUpdateSector = cluster.sector; + assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { - StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -230,7 +226,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +{ + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + return; + } + if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + return; + } + const GPUParam& GPUrestrict() param = merger.Param(); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad)))) { + *dEdxSubThresholdRow = iRow; + } + } + } + if (doAttach) { + AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -343,8 +363,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } - FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); - return 0; // ok + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -464,9 +483,12 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { + const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + const GPUCalibObjectsConst& GPUrestrict() calib = merger.GetConstantMem()->calibObjects; + const ClusterNative* GPUrestrict() clustersArray = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; if (acc || accAlt) { @@ -485,10 +507,16 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& relTime = relTime - CAMath::Round(relTime); const auto& cluster = clusters[ihit]; if (acc) { + if (dEdxSubThresholdRow != 255) { // Must be called before fillCluster, which flushes pending sub-threshold rows with the min charge of their ROC when the ROC changes + dEdx.fillSubThreshold(dEdxSubThresholdRow); + } dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); } if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { if (accAlt) { + if (dEdxSubThresholdRow != 255) { + dEdxAlt.fillSubThreshold(dEdxSubThresholdRow); + } dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); } } @@ -497,37 +525,36 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& } } -GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) +GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { constexpr float kDeg2Rad = M_PI / 180.f; - constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; } ConstrainSinPhi(); - if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } } *this = save; Alpha = saveAlpha; } - if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(kSectAngle / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; } } -GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) +GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& GPUrestrict() param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) { if (mirrorParameters) { prop.Mirror(inFlyDirection); @@ -554,7 +581,7 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, const bool markReject) { const int32_t ihitFirst = ihit; { @@ -575,12 +602,12 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); - if (rejectChi2) { + if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; @@ -630,7 +657,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float r2 = xx * xx + yy * yy; const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); if (r2 < rmax2) { - if (rejectChi2) { + if (markReject) { MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); } return -1; @@ -806,7 +833,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; - constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; @@ -821,7 +847,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger CADEBUG(printf("\tRotated: X %f Y %f Z %f SinPhi %f (Alpha %f / %f)\n", mP[0], mX, mP[1], mP[2], prop.GetAlpha(), prop.GetAlpha() + CAMath::Pi() / 2.f)); uint32_t maxTries = 100; while (true) { - while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(kSectAngle / 2.f) + 0.1f) { + while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { if (maxTries-- == 0) { return; } @@ -872,7 +898,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { - constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; @@ -907,7 +932,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr Y += dS * SinPhi + h4; Z += dS * mP[3]; SinPhi = newSinPhi; - if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { // printf("Abort, sector edge\n"); return; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index d738f09a91c15..f0ce761d41418 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,12 +155,13 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index c8d6b00d916cd..7ce2cb82a0a50 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -401,13 +401,11 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { constexpr float kDeg2Rad = M_PI / 180.f; - constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { - // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it - const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; + const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef)) { - if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef); } From efcc7842b3451772384cebe28e461c862cfea2f8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 21:35:02 +0100 Subject: [PATCH 14/60] GPU: Move some functionality to CAMath --- GPU/Common/GPUCommonMath.h | 1 + GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 9 +++------ GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 3 +-- GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h | 2 +- GPU/GPUTracking/qa/GPUQA.cxx | 12 ++++++------ GPU/GPUTracking/qa/genEvents.cxx | 4 ++-- GPU/GPUTracking/qa/genEvents.h | 4 ++-- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index c55b04ddc0743..62efdfb80bb35 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -122,6 +122,7 @@ class GPUCommonMath GPUd() constexpr static double Copysignd(double x, double y); GPUd() constexpr static float TwoPi() { return 6.2831853f; } GPUd() constexpr static float Pi() { return 3.1415927f; } + GPUd() constexpr static float Deg2Rad() { return Pi() / 180.f; } GPUd() constexpr static float Round(float x); GPUd() constexpr static float Floor(float x) { return GPUCA_CHOICE(floorf(x), floorf(x), floor(x)); } GPUd() static uint32_t Float2UIntReint(const float& x); diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 32fbe83357d6b..ac2560a45ef90 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -16,6 +16,7 @@ #define GPUTPCGEOMETRY_H #include "GPUCommonDef.h" +#include "GPUCommonMath.h" #include "DataFormatsTPC/Constants.h" #ifndef GPUCA_GPUCODE_DEVICE #include @@ -178,7 +179,7 @@ class GPUTPCGeometry return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } - GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } + GPUd() static constexpr float kSectAngle() { return 2 * CAMath::Pi() / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 612a99de4d2d6..ee70dd488a44e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -278,8 +278,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } - static constexpr float kDeg2Rad = M_PI / 180.f; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { return 1; // bad chi2 for the whole track, stop the fit @@ -527,13 +526,11 @@ GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& G GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { - constexpr float kDeg2Rad = M_PI / 180.f; - if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; @@ -547,7 +544,7 @@ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestric Alpha = saveAlpha; } if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 7ce2cb82a0a50..906e943bb1743 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -400,12 +400,11 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - constexpr float kDeg2Rad = M_PI / 180.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef); } diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h index 0d71b35e92c78..6ef8643215e6d 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h @@ -147,7 +147,7 @@ class trackInterface : public GPUTPCGMTrackParam GPUd() float getSnp() const { return GetSinPhi(); } GPUd() float getTgl() const { return GetDzDs(); } GPUd() float getQ2Pt() const { return GetQPt(); } - GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * M_PI - CAMath::ATan(getTgl())))); } + GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * CAMath::Pi() - CAMath::ATan(getTgl())))); } GPUd() float getPt() const { return CAMath::Abs(getQ2Pt()) > 0 ? CAMath::Abs(1.f / getQ2Pt()) : 99999.f; } GPUd() float getSigmaY2() const { return GetErr2Y(); } GPUd() float getSigmaZ2() const { return GetErr2Z(); } diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 689f9a36fc2fe..2e8e2c129a1fa 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -182,7 +182,7 @@ static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0 static const constexpr int32_t CONFIG_DASHED_MARKERS = 0; static const constexpr float AXES_MIN[5] = {-Y_MAX, -Z_MAX, 0.f, -ETA_MAX, PT_MIN}; -static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * M_PI, ETA_MAX, PT_MAX}; +static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * CAMath::Pi(), ETA_MAX, PT_MAX}; static const constexpr int32_t AXIS_BINS[5] = {51, 51, 144, 31, 50}; static const constexpr int32_t RES_AXIS_BINS[] = {1017, 113}; // Consecutive bin sizes, histograms are binned down until the maximum entry is 50, each bin size should evenly divide its predecessor. static const constexpr float RES_AXES[5] = {1., 1., 0.03, 0.03, 1.0}; @@ -1247,12 +1247,12 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx const mcInfo_t& info = GetMCTrack(i, iCol); additionalMCParameters& mc2 = mMCParam[iCol][i]; mc2.pt = std::sqrt(info.pX * info.pX + info.pY * info.pY); - mc2.phi = M_PI + std::atan2(-info.pY, -info.pX); + mc2.phi = CAMath::Pi() + std::atan2(-info.pY, -info.pX); float p = info.pX * info.pX + info.pY * info.pY + info.pZ * info.pZ; if (p < 1e-18) { mc2.theta = mc2.eta = 0.f; } else { - mc2.theta = info.pZ == 0 ? (M_PI / 2) : (std::acos(info.pZ / std::sqrt(p))); + mc2.theta = info.pZ == 0 ? (CAMath::Pi() / 2) : (std::acos(info.pZ / std::sqrt(p))); mc2.eta = -std::log(std::tan(0.5 * mc2.theta)); } if (mConfig.writeMCLabels) { @@ -1306,10 +1306,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } float alpha = std::atan2(info.y, info.x); - alpha /= M_PI / 9.f; + alpha /= CAMath::Pi() / 9.f; alpha = std::floor(alpha); - alpha *= M_PI / 9.f; - alpha += M_PI / 18.f; + alpha *= CAMath::Pi() / 9.f; + alpha += CAMath::Pi() / 18.f; float c = std::cos(alpha); float s = std::sin(alpha); diff --git a/GPU/GPUTracking/qa/genEvents.cxx b/GPU/GPUTracking/qa/genEvents.cxx index 38c2cf2914770..c1f33a1e0bbc9 100644 --- a/GPU/GPUTracking/qa/genEvents.cxx +++ b/GPU/GPUTracking/qa/genEvents.cxx @@ -202,8 +202,8 @@ int32_t genEvents::GenerateEvent(const GPUParam& param, const char* filename) double eta = gRandom->Uniform(-1.5, 1.5); double theta = 2 * std::atan(1. / exp(eta)); - double lambda = theta - M_PI / 2; - // double theta = gRandom->Uniform(-60,60)*M_PI/180.; + double lambda = theta - CAMath::Pi() / 2; + // double theta = gRandom->Uniform(-60,60)*CAMath::Pi()/180.; double pt = .08 * std::pow(10, gRandom->Uniform(0, 2.2)); double q = 1.; diff --git a/GPU/GPUTracking/qa/genEvents.h b/GPU/GPUTracking/qa/genEvents.h index ea209a0a46cb8..6bd6d4b370f64 100644 --- a/GPU/GPUTracking/qa/genEvents.h +++ b/GPU/GPUTracking/qa/genEvents.h @@ -16,7 +16,7 @@ #define GENEVENTS_H #include "GPUCommonDef.h" -#include +#include "GPUCommonMath.h" namespace o2::gpu { @@ -66,7 +66,7 @@ class genEvents uint32_t id; }; - const double mTwoPi = 2 * M_PI; + const double mTwoPi = 2 * CAMath::Pi(); const double mSectorDAngle = mTwoPi / 18.; const double mSectorAngleOffset = mSectorDAngle / 2; From b66abbb9941a893e1cfaba6ea5d31b9a99dc7524 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 00:11:27 +0100 Subject: [PATCH 15/60] GPU TPC: Store track position for interpolation also when not updating with new hit --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 133 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 4 files changed, 76 insertions(+), 65 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 49e0773abab1d..ddfefe8636af5 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -638,7 +638,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) +GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index bf67d027f6b05..e637c315f3ce6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -102,7 +102,7 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index ee70dd488a44e..7cc312fcdd4f2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -123,10 +123,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { @@ -140,31 +143,67 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; } lastPropagateRow = cluster.row; - if (currentClusterStatus) { - nMissed++; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all + prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection))) || // propagation fails nonetheless + retValProp) { // failed for other reason but rotation + CADEBUG(printf(" --- break-prop\n")); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - if (doInterpolate) { - FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); - } + NTolerated++; continue; } - - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + break; // bad chi2 for the whole track, stop the fit + } if ((uint32_t)interpolationIndex >= interpolation.size) { merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } auto& inter = interpolation.hit[interpolationIndex]; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit) { + if (iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } + if (allowChangeClusters) { + AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit + } + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + float tmpZ; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); + } + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + continue; + } + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -226,11 +265,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) { - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - return; - } if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } @@ -250,46 +286,20 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; // bad chi2 for the whole track, stop the fit - } - if (retValProp || sinPhiErr) { + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + CADEBUG(printf(" --- break-sinphi\n")); return 2; // Propagate failed or high incl angle } - CADEBUG(printf("\n")); if (crossCE) { if (param.rec.tpc.addErrorsCECrossing) { @@ -307,6 +317,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + if (param.rec.tpc.rebuildTrackInFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds + } } else { float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; @@ -317,15 +330,15 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = false; + bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 3) { - prop.InterpolateFill(yy, zz, &inter); - } else if (iWay == nWays - 2) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else if (!param.rec.tpc.rebuildTrackInFit) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { @@ -333,10 +346,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } } else { - rejectChi2 = GetNDF() > 0 && allowChangeClusters; + rejectChi2 = allowChangeClusters; } - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; } else { retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); @@ -367,7 +380,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } if (!retValInt) { nMissed++; @@ -473,7 +486,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -599,17 +612,17 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); + CADEBUG(printf("\t\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; } else { - CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + CADEBUG(printf("\t\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); return true; } }; @@ -1061,9 +1074,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - // clang-format off - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; } ); - // clang-format on + CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index f0ce761d41418..29b4f446965b4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,8 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From dcf4c67e0f3dd4dfdbe1cd44d5e7e84ce2338c68 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 10:39:27 +0100 Subject: [PATCH 16/60] GPU TPC: Extract CE crossing handling in function, and apply before storing interpolated position --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7cc312fcdd4f2..eebef43473413 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -97,7 +97,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -133,7 +133,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); if (dodEdx || doAttach || doInterpolate) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; @@ -177,9 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } auto& inter = interpolation.hit[interpolationIndex]; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { - prop.InterpolateFill(&inter); - } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { @@ -196,14 +193,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } + HandleCrossCE(param, cluster.sector, lastSector); + lastSector = cluster.sector; + + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + if (currentClusterStatus) { nMissed++; nMissed2++; continue; } - const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -218,11 +221,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - lastUpdateSector = cluster.sector; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { - StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { @@ -286,7 +288,23 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) +{ + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (sector < 18)); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -301,18 +319,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger return 2; // Propagate failed or high incl angle } - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 29b4f446965b4..c6f2d58cc9cca 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,9 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From 0d236ba8e0789afc6f2b4c9549f3a449cfbca2ed Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 11:16:51 +0100 Subject: [PATCH 17/60] GPU TPC: If value from other side of interpolation not stored, use only current value --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 34322048511ae..1912dff85382d 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -102,6 +102,8 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") +AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -137,7 +139,6 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 8ab663b5f791a..b3c1d02e99c8d 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1031,6 +1031,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index eebef43473413..98e4112cea6a8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -406,7 +406,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -414,19 +414,24 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre const float stepZ = rowData.HstepZ(); int32_t bin, ny, nz; - float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; + float ImP0, ImP1, ImC0, ImC2; + if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + } else { + ImP0 = mP[0]; + ImP1 = mP[1]; + ImC0 = mC[0]; + ImC2 = mC[2]; + } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); @@ -435,6 +440,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); From 7f582f88382a7f360320300c402211ca56524637 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 00:00:45 +0100 Subject: [PATCH 18/60] GPU TPC: Mark clusters of rebuilt track as high incli when interpolated in NoUpdateHighIncl region --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 6 ++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index ef5abc7e49a31..be9f447093e12 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2271,7 +2271,11 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); + if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { + flags |= GPUTPCGMMergedTrackHit::flagHighIncl; + } + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; written++; CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index d07a47d4fe981..66ff40f79ac3f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -88,7 +88,9 @@ class GPUTPCGMMerger : public GPUProcessor }; struct trackRebuildHelper { - bool reverse; + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; }; struct tmpSort { @@ -133,6 +135,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 98e4112cea6a8..5792987379458 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -64,6 +64,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } + if (param.rec.tpc.rebuildTrackInFit) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + } } const int32_t nWays = param.rec.tpc.nWays; @@ -206,6 +210,28 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + const bool inward = clusters[0].row > clusters[maxN - 1].row; + const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } + } + } + continue; + } + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); @@ -215,9 +241,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ covYYUpd = mC[0]; } else if (retValHit == 1) { break; - } else if (retValHit == 2) { - NTolerated++; - continue; } lastUpdateRow = cluster.row; @@ -311,14 +334,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - return 2; // Propagate failed or high incl angle - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { From 16c23f1370ef227757ed95ff780267b956e0c6b5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 18:39:10 +0100 Subject: [PATCH 19/60] GPU TPC: Pick up best hit from interpolation only, if current position not available --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 54 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index b3c1d02e99c8d..df74187a4f9d5 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1031,7 +1031,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; - rec.tpc.rebuildTrackMaxNonIntCov = 0.f; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; // TODO: Check if this yields a performance benefit } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 5792987379458..2ec863fdb6861 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -106,7 +106,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); @@ -123,7 +123,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; - interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; @@ -157,6 +156,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastPropagateRow = cluster.row; + auto& inter = interpolation.hit[interpolationIndex]; int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all @@ -166,6 +166,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + } continue; } // clang-format off @@ -173,6 +176,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; // bad chi2 for the whole track, stop the fit } @@ -180,12 +186,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } - auto& inter = interpolation.hit[interpolationIndex]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit @@ -240,6 +245,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; } @@ -413,7 +421,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -421,7 +429,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -430,9 +438,14 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre int32_t bin, ny, nz; float ImP0, ImP1, ImC0, ImC2; - if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; + if (interOnly) { + ImP0 = (float)inter.posY; + ImP1 = (float)inter.posZ + deltaZ; + ImC0 = (float)inter.errorY; + ImC2 = (float)inter.errorZ; + } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = (float)inter.posY - mP[0]; + const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); const float Ik00 = mC[0] * Iw0; @@ -517,14 +530,33 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } - if (nCandidates == 0) { + if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } } return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + for (; ihit >= 0; ihit--, interpolationIndex--) { + while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { + ihit--; + } + const auto& cluster = clusters[ihit]; + if (CAMath::Abs(cluster.row - lastRow) > 1) { + interpolationIndex -= CAMath::Abs(cluster.row - lastRow) - 1; + } + lastRow = cluster.row; + if (interpolationIndex < 0) { // TODO: Is this check needed? + return; + } + auto inter = interpolation.hit[interpolationIndex]; + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + } +} + +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index c6f2d58cc9cca..572dfe67c2330 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,8 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); From e45cc99acd7a08f65b9b7d95108b9b403ccbbe6d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 13:04:02 +0100 Subject: [PATCH 20/60] GPU TPC: Interpolate missing pad rows when rebuilding track --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index d8b14ecc61182..88388a4b9074f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -35,6 +35,8 @@ enum attachTypes { attachProtect = 0x80000000, struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; + GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } + GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2ec863fdb6861..47dcf98f583d3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -61,12 +61,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size - interpolation.hit[i].errorY = -1; + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + interpolation.hit[i].markInvalid(); } if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } } @@ -134,22 +133,24 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); uint8_t dEdxSubThresholdRow = 255; - bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - if (dodEdx || doAttach || doInterpolate) { + const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; - for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + uint8_t sector = lastSector; + + for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } - FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; @@ -233,6 +234,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; } } + // TODO: We can perhaps break here, if we pick up remaining rows } continue; } @@ -298,7 +300,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; @@ -317,6 +319,16 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se if (doAttach) { AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); } + if (doInterpolate) { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > 4.f) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + } else { + prop.InterpolateFill(&inter); + } + } } GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) @@ -363,10 +375,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (param.rec.tpc.mergerInterpolateErrors) { if (iWay == nWays - 2) { if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else { + if (inter.isValid()) { retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; } } } else if (iWay == nWays - 1) { @@ -429,7 +441,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -443,7 +455,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + } else if (inter.isValid()) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -520,7 +532,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } + } // TODO: Downweight non-interpolated candidates merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 572dfe67c2330..7371cfd8afdef 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -157,7 +157,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); From e5abada49fd059a25add62198c24efe421b959ba Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 14:07:16 +0100 Subject: [PATCH 21/60] Revert "GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future" This reverts commit 5c3d113f80687ecfcb3bf106b0fc7ad274fceda9. We can actually properly retry also for rebuilt tracks, we just run the retry on the last iteration after rebuilding. --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 4 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 + GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 62 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +- GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 8 files changed, 60 insertions(+), 30 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 1912dff85382d..b884c3052ad7e 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -157,6 +157,7 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 52cdcd5dee78f..596bc8f614666 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -248,6 +248,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } + if (param().rec.tpc.retryRefit) { + runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + } + runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index be9f447093e12..8b4bf329634cb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,13 +402,16 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } + if (mRec->GetParam().rec.tpc.retryRefit) { + computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 66ff40f79ac3f..ee63269b532fb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,6 +67,7 @@ class GPUTPCGMMerger : public GPUProcessor static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { + GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -140,6 +141,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } + GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -322,6 +324,7 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; + uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 083cdf76699a0..2ef0d0ba59748 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,9 +21,10 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { - const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); + const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 47dcf98f583d3..75e8786ff9186 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool finalFit = iWay == nWays - 1; ResetCovariance(); - prop.SetSeedingErrors(!(refit)); + prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); @@ -239,7 +239,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -256,7 +256,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -347,7 +347,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -371,23 +371,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { + bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); + if (!retryAttempt) { + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { rejectChi2 = true; } } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; - } + } else { + rejectChi2 = allowChangeClusters; } - } else { - rejectChi2 = allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { @@ -893,6 +895,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; + if (!merger.MergedTracks()[data.track].OK()) { + return; + } GPUTPCGMPropagator prop; prop.SetMaterialTPC(); prop.SetPolynomialField(&merger.Param().polynomialField); @@ -902,7 +908,6 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); @@ -1141,7 +1146,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) { if (!track.OK()) { return; @@ -1153,16 +1158,29 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { + for (uint32_t i = 0; i < track.NClusters(); i++) { + merger.Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + CADEBUG(printf("Track rejected, marking for retry\n")); + uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); + merger.RetryRefitIds()[nRefit] = iTrk; + return; + } + if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { + ok = false; + } + if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + track.SetOK(ok); if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 7371cfd8afdef..a2a677db8c5de 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,12 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 2e8e2c129a1fa..2038c4c84f707 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2968,7 +2968,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - if (mQATasks & taskTrackStatistics) { + if (mQATasks & taskTrackStatistics) { // TODO: Add plot for Chi2/NDF, and NCl correct and fake MC histograms // Process track statistic histograms float tmpMax = 0.; From c5153a5b49efa29e7e2a58e85aa106e9b7b1c296 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 11:06:13 +0100 Subject: [PATCH 22/60] GPU TPC: Relax chi2cut for single-pad clusters --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index e637c315f3ce6..3f1a74cfb6c26 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -108,13 +108,19 @@ class GPUTPCGMPropagator GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) { - if (chiY > 9.f || chiZ > 9.f) { // TODO: Check how a track can have chi2/ncl > 18 + if (chiY > 9.f || chiZ > 9.f) { return 2; } - if ((chiY > 6.25f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + if (chiZ > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared | GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { return 2; } - if ((chiY > 1.f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { + if (chiY > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + return 2; + } + if (chiY > 1.f && (clusterState & GPUTPCGMMergedTrackHit::flagEdge)) { + return 2; + } + if (chiY > 2.f && (clusterState & GPUTPCGMMergedTrackHit::flagSingle)) { return 2; } return 0; From 3a6fef9fd9f5324747c760a651bc57b26781449e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:41:14 +0100 Subject: [PATCH 23/60] GPU TPC: Clean up and fix resetT0 code --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 22 +++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 15 ++----------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 75e8786ff9186..9bce56b33b057 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -85,7 +85,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! } - int32_t resetT0 = initResetT0(); + int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; @@ -239,9 +239,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { - DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + resetT0 = 20; + } + ihitStart = ihit; interpolatedStart = interpolationIndex; N++; @@ -347,7 +353,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -412,12 +418,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger lastUpdateX = mX; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { @@ -570,7 +570,7 @@ GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrest } } -GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a2a677db8c5de..6b5322a8ec1cf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,8 +155,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); - GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); @@ -229,8 +229,6 @@ class GPUTPCGMTrackParam } private: - GPUd() int32_t initResetT0(); - float mX; // x position float mTOffset; // Z offset with early transform, T offset otherwise float mP[5]; // 'active' track parameters: Y, Z, SinPhi, DzDs, q/Pt @@ -248,15 +246,6 @@ struct GPUTPCGMLoopData { uint8_t outwards; }; -GPUdi() int32_t GPUTPCGMTrackParam::initResetT0() -{ - const float absQPt = CAMath::Abs(mP[4]); - if (absQPt < (150.f / 40.f)) { - return 150.f / 40.f; - } - return CAMath::Max(10.f, 150.f / mP[4]); -} - GPUdi() void GPUTPCGMTrackParam::ResetCovariance() { mC[0] = 100.f; From 6335a9fb13c9b716fe828c9c2c0e9e58142219ca Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:42:27 +0100 Subject: [PATCH 24/60] GPU TPC: Don't break for high sinPhi if only the linearization model is out of range, but instead reinitialize the model --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 9bce56b33b057..2e499bd501b5b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -217,26 +217,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - NTolerated++; + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); - if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { - if (inward ^ (iWay & 1)) { - if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; - } - } else { - if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + if (mNDF > 10 && CAMath::Abs(mP[2]) < maxSinForUpdate && markHighIncl) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } else { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } } + // TODO: We can perhaps break here, if we pick up remaining rows } - // TODO: We can perhaps break here, if we pick up remaining rows + continue; } - continue; } int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); From 4b896d4d984e44e1d35ffb36d6a130176dd7e54e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 18 Nov 2025 15:37:23 +0100 Subject: [PATCH 25/60] GPU: Add option to steer which attachment steps to run during rebuild --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 7 +++--- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 22 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++--------- .../Standalone/Benchmark/standalone.cxx | 2 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b884c3052ad7e..212cdbdca425e 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,13 +139,14 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") -AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableMarkAdjacent = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") @@ -168,7 +169,7 @@ AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge cluste AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet hand AddOptionArrayRTC +AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 8b4bf329634cb..5b71478facb0d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2145,16 +2145,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, continue; } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; - int lastRow = -1; - for (uint32_t j = 0; j < trk.NClusters(); j++) { - const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; - lastRow = cl.row; + if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2e499bd501b5b..46f3562177eb0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -135,8 +135,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); + const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -167,7 +167,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later } continue; @@ -177,7 +177,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; // bad chi2 for the whole track, stop the fit @@ -190,15 +190,16 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit } } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + } + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } @@ -258,7 +259,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; @@ -267,7 +268,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -771,7 +772,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); - if (param.rec.tpc.disableRefitAttachment & 1) { + if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 38ca9adb3d242..652ec5ccc23f2 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -538,7 +538,7 @@ int32_t SetupReconstruction() if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { procSet.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableRefitAttachment = 0xFF; + recSet.tpc.disableMarkAdjacent = 0xFF; recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); recAsync->SetSettings(&grp, &recSet, &procSet, &steps); From cf25b11eb7a193c470959ca6df9f74a123a75a3c Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 19:19:38 +0100 Subject: [PATCH 26/60] GPU TPC: Extrapolate track inward and outward when rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 6 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 114 +++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 212cdbdca425e..36631002034f2 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,7 +139,7 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") @@ -168,7 +168,9 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") - +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 46f3562177eb0..c40236dca1b71 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,6 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -217,7 +218,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); @@ -268,16 +268,120 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { - StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); - } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); } else { deltaZ = 0.f; } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; + int32_t sector = lastSector; + uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; + uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + prop.SetTrack(this, prop.GetAlpha()); + for (uint32_t iRow = lastPropagateRow + up; iRow < GPUTPCGeometry::NROWS; iRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + bool fail = false; + for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { + float tmpX, tmpY, tmpZ; + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + fail = true; + break; + } + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { + fail = true; + break; + } + if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + fail = true; + break; + } + if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { + if (iAttempt) { + fail = true; + break; + } + const int32_t sectorSide = sector >= (int8_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; + if (mP[0] > 0) { + if (++sector >= sectorSide + 18) { + sector -= 18; + } + } else { + if (--sector < sectorSide) { + sector += 18; + } + } + } + } + if (fail) { + break; + } + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + gputpcgmmergertypes::InterpolationErrorHit inter; + inter.markInvalid(); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (candidate.id >= 2) { + lastExtrapolateRow = iRow; + float err2Y, err2Z, xx, yy, zz; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { + candidate.best = -1; + break; + } + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(cl.qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / cl.qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + const uint8_t clusterState = cl.getFlags(); + prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); + uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd != 0) { + candidate.best = -1; + break; + } + if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { + lastGoodRow = iRow; + consecGoodRowsMissing = 0; + } + rowGapActive = rowGapTotal = 0; + } else { + if (++missingRowsTotal > param.rec.tpc.rebuildTrackExtrMaxMissingRows) { + break; + } + if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { + break; + } + if (consecGoodRowsMissing++ >= param.rec.tpc.rebuildTrackExtrConsecGoodRowsMaxGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + } + } + } + if (lastGoodRow != lastExtrapolateRow) { + for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + candidate.best = -1; + } + } + } + + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + } + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { From ff325d296aac31e0b290de471fdc31b6dd528a75 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Dec 2025 12:05:46 +0100 Subject: [PATCH 27/60] GPU: Add debug option to create temporary MC labels for collected merged tracks --- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 4 +-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 13 +++++-- .../GPUTrackingLinkDef_O2_DataTypes.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 ++++++++++++++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 13 +++++++ GPU/GPUTracking/qa/GPUQAHelper.h | 11 ++++++ 8 files changed, 76 insertions(+), 6 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index be8b5eed49156..b7983a8c81ec1 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -216,10 +216,10 @@ int32_t GPUReconstructionCPU::ExitDevice() int32_t GPUReconstructionCPU::RunChains() { mMemoryScalers->temporaryFactor = 1.; - if (GetProcessingSettings().memoryScalingFuzz) { + if (GetProcessingSettings().debug.memoryScalingFuzz) { static std::mt19937 rng; static std::uniform_int_distribution dist(0, 1000000); - uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz; + uint64_t fuzzFactor = GetProcessingSettings().debug.memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().debug.memoryScalingFuzz; GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor); mMemoryScalers->fuzzScalingFactor(fuzzFactor); } diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 36631002034f2..2b28500646f85 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -32,7 +32,7 @@ BeginNamespace(o2::gpu) // Reconstruction parameters for TPC, no bool in here !!! BeginSubConfig(GPUSettingsRecTPC, tpc, configStandalone.rec, "RECTPC", 0, "Reconstruction settings", rec_tpc) -AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") +AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") // TODO: Sort these options automatically for parameter size AddOptionRTC(hitPickUpFactor, float, 1.f, "", 0, "multiplier for the combined cluster+track error during track following") AddOptionRTC(hitSearchArea2, float, 2.f, "", 0, "square of maximum search road of hits during seeding") AddOptionRTC(neighboursSearchArea, float, 3.f, "", 0, "area in cm for the search of neighbours, for z only used if searchWindowDZDR = 0") @@ -330,6 +330,13 @@ AddOption(tpcDecodingSafetyBuffer, uint16_t, 1000, "", 0, "Scaling Factor: safet AddHelp("help", 'h') EndConfig() +// Debug Settings +BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) +AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) +AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddHelp("help", 'h') +EndConfig() + // Settings steering the processing once the device was selected, only available on the host BeginSubConfig(GPUSettingsProcessing, proc, configStandalone, "PROC", 0, "Processing settings", proc) AddOption(deviceNum, int32_t, -1, "gpuDevice", 0, "Set GPU device to use (-1: automatic, -2: for round-robin usage in timeslice-pipeline)") @@ -359,7 +366,6 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strateg AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(uint64_t(0))) AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(uint64_t(0))) AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers") -AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations") AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n") AddOption(registerStandaloneInputMemory, bool, false, "registerInputMemory", 0, "Automatically register input memory buffers for the GPU") @@ -406,7 +412,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") -AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") +AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") AddOption(metalOverrideSourceBuildFlags, std::string, "", "", 0, "Override Metal build flags for compilation from source, put a space for empty options") @@ -431,6 +437,7 @@ AddSubConfig(GPUSettingsProcessingRTCtechnical, rtctech) AddSubConfig(GPUSettingsProcessingParam, param) AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) +AddSubConfig(GPUSettingsProcessingDebug, debug) AddHelp("help", 'h') EndConfig() #endif // !__OPENCL__ && !__METAL__ diff --git a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h index 21e495e3ca6c6..2628821bb3ca1 100644 --- a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h +++ b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h @@ -33,6 +33,7 @@ #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingRTCtechnical + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingNNclusterizer + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingScaling + ; +#pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingDebug + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplay + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayInternal + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayLight + ; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 596bc8f614666..6fe838d939dbb 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -176,6 +176,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{1, -WarpSize(), 0, deviceType}}, 1); runKernel({{1, -WarpSize(), 0, deviceType}}, 1); } + if (!doGPU && GetProcessingSettings().debug.mergerMCLabels) { + Merger.CreateMCLabels(1, 1, 0, 0); + } DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingCollectedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpCollected, *mDebugFile); if (param().rec.tpc.mergeCE) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 5b71478facb0d..53904b8f18c0c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -148,6 +148,7 @@ using namespace o2::gpu::internal; #include "GPUQA.h" #include "GPUMemorySizeScalers.h" +#include "GPUQAHelper.h" GPUTPCGMMerger::GPUTPCGMMerger() { @@ -164,7 +165,7 @@ GPUTPCGMMerger::GPUTPCGMMerger() } // DEBUG CODE -#if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) +#if defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC #include "GPUQAHelper.h" template @@ -438,6 +439,9 @@ void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) void* GPUTPCGMMerger::SetPointersOutput(void* mem) { computePointerWithAlignment(mem, mMergedTracks, mNMaxTracks); + if (mRec->GetProcessingSettings().debug.mergerMCLabels) { + computePointerWithAlignment(mem, mMergedTrackMC, mNMaxTracks); + } if (mRec->GetParam().dodEdxEnabled) { computePointerWithAlignment(mem, mMergedTracksdEdx, mNMaxTracks); if (mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMask != mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMaskAlt) { @@ -547,6 +551,34 @@ int32_t GPUTPCGMMerger::CheckSectors() return 0; } +void GPUTPCGMMerger::CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + const o2::tpc::ClusterNativeAccess* GPUrestrict() clusters = GetConstantMem()->ioPtrs.clustersNative; + if (clusters == nullptr || clusters->clustersMCTruth == nullptr) { + return; + } + if (mMergedTrackMC == nullptr) { + return; + } + + auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + labelAssigner.reset(); + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + if (cl.state & GPUTPCGMMergedTrackHit::flagReject) { + continue; + } + labelAssigner.addLabel(cl.num); + } + mMergedTrackMC[i] = labelAssigner.computeLabel(); + } +} + #endif // GPUCA_GPUCODE GPUd() void GPUTPCGMMerger::ClearTrackLinks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, bool output) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index ee63269b532fb..774634f34bd1a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -123,6 +123,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() int32_t NMergedTracks() const { return mMemory->nMergedTracks; } GPUhdi() const GPUTPCGMMergedTrack* MergedTracks() const { return mMergedTracks; } + GPUhdi() const o2::MCCompLabel* MergedTrackMC() const { return mMergedTrackMC; } GPUhdi() GPUTPCGMMergedTrack* MergedTracks() { return mMergedTracks; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdx() const { return mMergedTracksdEdx; } GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } @@ -214,6 +215,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -294,6 +296,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + o2::MCCompLabel* mMergedTrackMC = nullptr; trackCluster* mClusterCandidates = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c40236dca1b71..2b37094a53ee4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -14,6 +14,7 @@ #define GPUCA_CADEBUG 0 #define DEBUG_SINGLE_TRACK -1 +// #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" #include "GPUTPCGMTrackParam.h" @@ -39,6 +40,11 @@ #include "AliHLTTPCClusterMCData.h" #endif +#ifndef GPUCA_GPUCODE +#include "SimulationDataFormat/ConstMCTruthContainer.h" +#include "SimulationDataFormat/MCCompLabel.h" +#endif + #ifndef GPUCA_GPUCODE_DEVICE #include #include @@ -277,6 +283,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; + CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -325,6 +332,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; if (candidate.id >= 2) { lastExtrapolateRow = iRow; +#if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) + if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); + } +#endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); diff --git a/GPU/GPUTracking/qa/GPUQAHelper.h b/GPU/GPUTracking/qa/GPUQAHelper.h index dcbe7a94711aa..6712fd0a603ed 100644 --- a/GPU/GPUTracking/qa/GPUQAHelper.h +++ b/GPU/GPUTracking/qa/GPUQAHelper.h @@ -162,6 +162,17 @@ static inline auto GPUTPCTrkLbl(const AliHLTTPCClusterMCLabel* x, Args... args) } } +template +static inline bool GPUTPCTrkLblSearch(const T& clusterLabels, const MCCompLabel& trkLabel) +{ + for (const auto& clLabel : clusterLabels) { + if (trkLabel.compare(clLabel) >= 0) { + return true; + } + } + return false; +} + } // namespace gpu } // namespace o2 From d11f330b2313b8d86b83081cb0e07e62ccf96672 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 10 Nov 2026 16:16:25 +0100 Subject: [PATCH 28/60] GPU TPC: Correctly handle rowGap and fit Failures when interpolating / extrapolating --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2b37094a53ee4..480b67d994196 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -108,6 +108,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N = 0; uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; + bool fitBroken = false; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -139,11 +140,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); + if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); + fitBroken = true; + break; + } + uint8_t dEdxSubThresholdRow = 255; - if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + if (rowGap) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); - const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); + const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); + const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides + const uint8_t doInterpolate = ((param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -158,9 +168,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + if (crossCE && doInterpolate == 2) { // Track crosses the CE within the row gap, search also on the other side, the sector there has the same alpha + FitAddRow(iRow, sector >= GPUTPCGeometry::NSECTORS / 2 ? sector - GPUTPCGeometry::NSECTORS / 2 : sector + GPUTPCGeometry::NSECTORS / 2, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, false, false, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + } } } - interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + interpolationIndex += rowGap * wayDirection; } lastPropagateRow = cluster.row; @@ -187,14 +200,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } + fitBroken = true; break; // bad chi2 for the whole track, stop the fit } - if ((uint32_t)interpolationIndex >= interpolation.size) { - merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); - break; - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { @@ -268,6 +277,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } + fitBroken = true; break; } @@ -280,10 +290,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; - CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); + CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -320,6 +330,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ sector += 18; } } + if (prop.RotateToAlpha(param.Alpha(sector))) { // GetPropagatedYZ / InverseTransformYZtoX of the next attempt need Y in the frame of the new sector + fail = true; + break; + } } } if (fail) { From ef5332e921bba1939fbc19064d727aa979e3fca0 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 10 Nov 2026 20:43:10 +0100 Subject: [PATCH 29/60] GPU TPC: Improve track extrapolation for better efficiency / less fake rate --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 14 ++- .../Global/GPUChainTrackingMerger.cxx | 10 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 11 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 106 +++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 5 files changed, 100 insertions(+), 43 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 2b28500646f85..f0257f2032db0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -104,6 +104,13 @@ AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of sha AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") +AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") +AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") +AddOptionRTC(rebuildTrackAltCandidateMaxErrFactor, float, 2.f, "", 0, "Alternative cluster candidates of a row are used only if their distance^2 is below this factor times the distance^2 of the best candidate") +AddOptionRTC(rebuildTrackAltCandidateGoodErr, float, 0.25f, "", 0, "Alternative cluster candidates of a row with distance^2 below this value are always used") +AddOptionRTC(rebuildTrackErrFactorOneSided, float, 1.5f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found with the track position from only one side during rebuild") +AddOptionRTC(rebuildTrackErrFactorExtrapolation, float, 1.f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found by track extrapolation during rebuild") +AddOptionRTC(rebuildTrackMaxInterpolationDistY, float, 4.f, "", 0, "Max distance in Y between the track positions from both sides to use their interpolation for the cluster search in rows without cluster during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -168,9 +175,12 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") -AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 20, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 8, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") +AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") +AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 6fe838d939dbb..9156b7700af9f 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -263,10 +263,12 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel(GetGridAuto(0, deviceType)); if (param().rec.tpc.mergeLoopersAfterburner) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0, deviceType) : GetGridAuto(0, deviceType)); - if (doGPU) { - TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0); - SynchronizeStream(0); // TODO: could probably synchronize on an event after runKernel - } + } + if (doGPU) { + TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0); + SynchronizeStream(0); // TODO: could probably synchronize on an event after runKernel + } + if (param().rec.tpc.mergeLoopersAfterburner) { runKernel(GetGridAuto(0, deviceType)); runKernel(doGPU ? GetGrid(Merger.Memory()->nLooperMatchCandidates, 0, deviceType) : GetGridAuto(0, deviceType)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingLoopers, Merger, &GPUTPCGMMerger::DumpLoopers, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 53904b8f18c0c..c477847213003 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2184,7 +2184,7 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { candidates[0].id = cl.num + 2; - candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly + candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly. Tracks with NDF <= 0 get no fallback in rows with candidates, and get those back only via the shared recovery candidates[0].weight = cl.state; candidates[0].sector = cl.sector; lastRow = cl.row; @@ -2204,15 +2204,14 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2) { - const int32_t nCl = trk.NClusters(); - float weightf = nCl * (nCl * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); candidates[iteration].weight = weight; CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); } else { - candidates[0].best = -1; + candidates[0].best = -2; } } } @@ -2259,7 +2258,7 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations } - } else if (candidates[0].best == 0 && candidates[0].id >= 2) { + } else if (candidates[0].best != -1 && candidates[0].id >= 2) { sharedRowCandidates[nSharedRowCandidates++] = j; } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 480b67d994196..97ba5b6aa964f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -83,6 +83,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; + // TODO: Could do 5 ways and interpolate again on the rebuilt clusters and reject in the final way based on that (and get the outer param from the rebuilt clusters), but costs 2 more ways for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! @@ -188,7 +189,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ nMissed2++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); } continue; } @@ -290,23 +291,25 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; - uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; - uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; + uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit + float segmentStartChi2 = mChi2; prop.SetTrack(this, prop.GetAlpha()); - for (uint32_t iRow = lastPropagateRow + up; iRow < GPUTPCGeometry::NROWS; iRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + uint32_t iExtrapolatedRow = lastPropagateRow + up; + for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers bool fail = false; for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iExtrapolatedRow), tmpY, tmpZ)) { fail = true; break; } - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iExtrapolatedRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { fail = true; break; @@ -339,13 +342,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (fail) { break; } - CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); - float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (candidate.id >= 2) { - lastExtrapolateRow = iRow; + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, true); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + bool found = candidate.id >= 2; + if (found) { + lastExtrapolateRow = iExtrapolatedRow; #if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); @@ -354,7 +358,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ #endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { candidate.best = -1; break; @@ -365,16 +369,30 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; invAvgCharge *= invAvgCharge; const uint8_t clusterState = cl.getFlags(); - prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + prop.GetErr2(err2Y, err2Z, param, zz, iExtrapolatedRow, clusterState, sector, time, invAvgCharge, invCharge); CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); - uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); - CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (consecGoodRows == 0) { // New segment + segmentStartChi2 = mChi2; + segmentRows = 0; + } + int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd != 0) { candidate.best = -1; - break; + if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { + break; + } + found = false; // Cluster rejected by chi2, handle as missing row + } else { + segmentRows++; } + } + if (found) { + lastFoundRow = iExtrapolatedRow; if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { - lastGoodRow = iRow; + if (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows) { + lastGoodRow = iExtrapolatedRow; // TODO: Could optionally store the track state here, and revert to the good state in case we reject further clusters + } consecGoodRowsMissing = 0; } rowGapActive = rowGapTotal = 0; @@ -385,8 +403,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { consecGoodRows = consecGoodRowsMissing = 0; } - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iExtrapolatedRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iExtrapolatedRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iExtrapolatedRow, pad))) { if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { break; } @@ -396,6 +414,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } + if (iExtrapolatedRow >= GPUTPCGeometry::NROWS && consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRowsEdge && (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows)) { // Reached the inner / outer end of the TPC, a shorter last segment is sufficient + lastGoodRow = lastFoundRow; + } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; @@ -464,7 +485,7 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } if (doInterpolate) { if (doInterpolate == 2) { - if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > 4.f) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { inter.markInvalid(); } FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); @@ -572,7 +593,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, bool extrapolation) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -580,7 +601,13 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + const float maxNonIntCov = param.rec.tpc.rebuildTrackMaxNonIntCov; + bool useInter = inter.isValid(); + if (useInter && interOnly) { // One-sided from the other direction, same limit as for the current track position + useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; + } + const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; + if (rowData.NHits() && (useInter || useCurrent)) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -594,7 +621,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.isValid()) { + } else if (useInter) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -614,21 +641,36 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + float xMoved = mX, yMoved = mP[0], zMoved = mP[1], sinPhiMoved = prop.GetSinPhi0(); + if (interOnly) { // Track is not at this row: estimate track position and angle there (for the cluster errors) from the interpolated position and the helix of the current linearization + float sA, cA; + CAMath::SinCos(param.Alpha(sector) - prop.GetAlpha(), sA, cA); + xMoved = GPUTPCGeometry::Row2X(row); + yMoved = ImP0; + zMoved = ImP1; + const float sinPhi = prop.GetSinPhi0() - prop.GetQPt0() * prop.GetBz(mX, mP[0], mP[1]) * (xMoved * cA - yMoved * sA - mX); // at the X of the row in the current track frame + if (CAMath::Abs(sinPhi) < constants::MAX_SIN_PHI) { + sinPhiMoved = sinPhi * cA - CAMath::Copysign(CAMath::Sqrt(1.f - sinPhi * sinPhi), prop.GetCosPhi0()) * sA; + } + sinPhiMoved = CAMath::Clamp(sinPhiMoved, -constants::MAX_SIN_PHI, constants::MAX_SIN_PHI); + } + int32_t nCandidates = 0; while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + param.GetClusterErrors2(sector, row, zMoved, sinPhiMoved, mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(ImC0))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(ImC2))); const float tubeY = CAMath::Sqrt(sy2); const float tubeZ = CAMath::Sqrt(sz2); rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + const float errFactor = extrapolation ? param.rec.tpc.rebuildTrackErrFactorExtrapolation : ((interOnly || !useInter) ? param.rec.tpc.rebuildTrackErrFactorOneSided : 1.f); // Downweight candidates not found by two-sided interpolation const int32_t nBinsY = rowData.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); @@ -653,13 +695,17 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); invAvgCharge *= invAvgCharge; - prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + if (interOnly) { + GPUTPCGMPropagator::GetErr2(err2YA, err2ZA, param, sinPhiMoved, mP[3], zMoved, xMoved, yMoved, row, clflags, sector, time, invAvgCharge, invCharge, false); + } else { + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + } const float Jw0 = 1.f / (ImC0 + err2YA); const float Jw2 = 1.f / (ImC2 + err2ZA); const float chi2Y = Jw0 * dy * dy; const float chi2Z = Jw2 * dz * dz; bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; + const float err = (param.rec.tpc.rebuildTrackCandidateErrChi2 ? (chi2Y + chi2Z) : (dy * dy + dz * dz)) * errFactor; if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { @@ -1304,7 +1350,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() } CADEBUG(printf("Track rejected, marking for retry\n")); uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); - merger.RetryRefitIds()[nRefit] = iTrk; + merger.RetryRefitIds()[nRefit] = iTrk; // TODO: With rebuildTrackInFit the retry runs on the rebuilt clusters, so it cannot recover tracks that lost their clusters in the rebuild return; } if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 6b5322a8ec1cf..4619db960981d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,7 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, bool extrapolation = false); GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use From 1b0486ff0b5ac01404eb159f847e193efd0d8e61 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 11 Nov 2026 13:58:59 +0100 Subject: [PATCH 30/60] GPU TPC: Let fallback clusters compete for ambiguity solving as all other clusters --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index f0257f2032db0..786fc4f4ce709 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -106,6 +106,8 @@ AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") +AddOptionRTC(rebuildTrackFallbackErr, float, 1.f, "", 0, "Error (distance^2) assigned to the original clusters of a track used as fallback candidates in rows that could not be searched during rebuild") +AddOptionRTC(rebuildTrackFallbackTrackChi2, float, 4.f, "", 0, "Chi2 per cluster assumed for the hit weight of tracks without successful fit (NDF <= 0) during rebuild") AddOptionRTC(rebuildTrackAltCandidateMaxErrFactor, float, 2.f, "", 0, "Alternative cluster candidates of a row are used only if their distance^2 is below this factor times the distance^2 of the best candidate") AddOptionRTC(rebuildTrackAltCandidateGoodErr, float, 0.25f, "", 0, "Alternative cluster candidates of a row with distance^2 below this value are always used") AddOptionRTC(rebuildTrackErrFactorOneSided, float, 1.5f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found with the track position from only one side during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index c477847213003..5b40560d9b61a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2178,15 +2178,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + const uint32_t nMaxCandidates = Param().rec.tpc.rebuildTrackInFitClusterCandidates; int lastRow = -1; for (uint32_t j = 0; j < trk.NClusters(); j++) { const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly. Tracks with NDF <= 0 get no fallback in rows with candidates, and get those back only via the shared recovery - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates]; + if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error + for (uint32_t k = 0; k < nMaxCandidates; k++) { + if (candidates[k].id == 0) { + candidates[k] = {.id = cl.num + 2, .row = cl.row, .sector = cl.sector, .error = Param().rec.tpc.rebuildTrackFallbackErr, .weight = 0, .best = 0}; + break; + } + } lastRow = cl.row; } } @@ -2198,15 +2201,15 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, { for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { const auto& trk = mMergedTracks[i]; - if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + if (!trk.OK()) { continue; } for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster - float weightf = (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); - int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2) / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster + int32_t weight = weightf > 0.f ? (weightf > 2e9f ? 2e9f : (int32_t)weightf) : 0; candidates[iteration].weight = weight; CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); @@ -2222,7 +2225,7 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads { for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { auto& trk = mMergedTracks[i]; - if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + if (!trk.OK()) { continue; } for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { @@ -2250,12 +2253,12 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads uint8_t nSharedRowCandidates = 0; for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + if (candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { candidates[0].best = nIter; } if (candidates[0].best > 0) { nHits++; - if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { + if (candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations } } else if (candidates[0].best != -1 && candidates[0].id >= 2) { @@ -2300,11 +2303,6 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best > 0) { auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; - if (candidates[0].best == 128) { - outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; - written++; - continue; - } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); From c034f321ba30cb45bbb71ba1a20df3d5bfc1378d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 11 Nov 2026 18:55:59 +0100 Subject: [PATCH 31/60] GPU TPC: Add debug option to dump merged track ncl to text file --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 2 +- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.h | 3 ++ .../GPUChainTrackingDebugAndProfiling.cxx | 23 +++++++++++ .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 5 ++- .../Standalone/Benchmark/CMakeLists.txt | 1 + .../Standalone/scripts/compareTrackNcl.py | 40 +++++++++++++++++++ 8 files changed, 76 insertions(+), 2 deletions(-) create mode 100755 GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index d1e96d6df9288..43af7a2cad438 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -295,7 +295,7 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice() if (GetProcessingSettings().deterministicGPUReconstruction && GetProcessingSettings().debugLevel >= 6) { mProcessingSettings->nTPCClustererLanes = 1; } - if (GetProcessingSettings().createO2Output > 1 && GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) { + if (GetProcessingSettings().createO2Output > 1 && ((GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) || GetProcessingSettings().debug.mergerDumpTrackNCl)) { mProcessingSettings->createO2Output = 1; } if (!GetProcessingSettings().createO2Output || !IsGPU()) { diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 786fc4f4ce709..b8c25d4a87931 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -346,6 +346,7 @@ EndConfig() BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddOption(mergerDumpTrackNCl, bool, false, "", 0, "Dump file with nCl per merged track fit") AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 1507147bbafda..92eda50194c74 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -322,6 +322,9 @@ class GPUChainTracking : public GPUChain int32_t RunTRDTrackingInternal(); uint32_t StreamForSector(uint32_t sector) const; + // Debug + void DoDebugMergerNClDump(const GPUTPCGMMerger& merger); + std::mutex mMutexUpdateCalib; std::unique_ptr mPipelineFinalizationCtx; GPUChainTrackingFinalContext* mPipelineNotifyCtx = nullptr; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index e45aeb4ba9246..13369405ac519 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -419,3 +419,26 @@ void GPUChainTracking::DoDebugRawDump() mRec->DumpSettings((dirName + "/").c_str()); DumpData((dirName + "/event.0.dump").c_str(), &ioPtrs); } + +void GPUChainTracking::DoDebugMergerNClDump(const GPUTPCGMMerger& merger) +{ + if (GetProcessingSettings().debug.mergerDumpTrackNCl) { +#ifndef GPUCA_DETERMINISTIC_MODE + GPUError("GPUCA_DETERMINISTIC_MODE not enabled, track dump will not be deterministic"); +#endif + if (!GetProcessingSettings().deterministicGPUReconstruction) { + GPUError("deterministicGPUReconstruction not enabled, track dump will not be deterministic"); + } + if (GetRecoStepsGPU() & RecoStep::TPCMerging) { + SynchronizeGPU(); + } + FILE* fout = fopen("merger_tracks_ncl.txt", "w+"); + if (fout == nullptr) { + throw std::runtime_error("Error Opening Debug File"); + } + for (int32_t i = 0; i < merger.NMergedTracks(); i++) { + fprintf(fout, "Track %7d NCl %5u\n", i, merger.MergedTracks()[i].OK() ? merger.MergedTracks()[i].NClustersFitted() : 0u); + } + fclose(fout); + } +} diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 9156b7700af9f..6786f7fc927e9 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,6 +235,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); } + // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); @@ -306,6 +307,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (GetProcessingSettings().keepDisplayMemory && !GetProcessingSettings().keepAllMemory) { TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger, -1, true); } + DoDebugMergerNClDump(Merger); + // mRec->SetNActiveThreads(-1); mRec->ReturnVolatileDeviceMemory(); mRec->PopNonPersistentMemory(RecoStep::TPCMerging, qStr2Tag("TPCMERGE")); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 97ba5b6aa964f..c3d254a0b9cc2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -37,6 +37,9 @@ #ifdef GPUCA_CADEBUG_ENABLED #include "GPUSettings.h" +#include +#include +#include #include "AliHLTTPCClusterMCData.h" #endif @@ -1335,7 +1338,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); + CADEBUG(if (DEBUG_SINGLE_TRACK != -1) { static const std::set dbgTracks = []() { std::set m; if (const char* e = DEBUG_SINGLE_TRACK == -2 ? getenv("DEBUG_TRACK") : nullptr) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } else { m.insert(DEBUG_SINGLE_TRACK); } return m; }(); if (!dbgTracks.count(iTrk)) { track.SetNClusters(0); track.SetOK(0); return; } }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt b/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt index d3bef7c0be1f6..efd165047fb7f 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt +++ b/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt @@ -32,6 +32,7 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") endif() install(DIRECTORY ../tools DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU) +install(DIRECTORY ../scripts DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU USE_SOURCE_PERMISSIONS) install(DIRECTORY ../../Definitions/Parameters/ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/tools FILES_MATCHING REGEX "\\.(python|sh|cmake)") target_compile_definitions(${targetName} PRIVATE $) diff --git a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py new file mode 100755 index 0000000000000..01ff969cf17a6 --- /dev/null +++ b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright 2019-2026 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +# Usage: compareTrackNcl.py FILE1 FILE2, with files of lines "Track NCl " (from --DEBUGmergerDumpTrackNCl). +# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with DEBUG_SINGLE_TRACK -2 in GPUTPCGMTrackParam.cxx, followed by some statistics. + +import sys + + +def readNCl(fileName): + ncl = {} + with open(fileName) as f: + for line in f: + cols = line.split() + if len(cols) >= 4 and cols[0] == "Track" and cols[2] == "NCl": + ncl[int(cols[1])] = int(cols[3]) + return ncl + + +if len(sys.argv) != 3: + sys.exit("Usage: " + sys.argv[0] + " FILE1 FILE2") +ncl1, ncl2 = readNCl(sys.argv[1]), readNCl(sys.argv[2]) +common = [i for i in sorted(ncl1) if i in ncl2] +ids = [str(i) for i in common if 2 * ncl1[i] < ncl2[i]] +print('DEBUG_TRACK="' + "\n".join(",".join(ids[i:i + 30]) for i in range(0, len(ids), 30)) + '"') +print() # Statistics as shell comments, so that the output can still be passed to eval +print("# Tracks in total (present in both files):", len(common)) +for text, n in (("Tracks with 0 clusters in file 2", sum(1 for i in common if ncl2[i] == 0)), + ("Tracks with 0 clusters in file 1 and > 0 in file 2", sum(1 for i in common if ncl1[i] == 0 and ncl2[i] > 0)), + ("Tracks with NCl(file 1) < 0.5 * NCl(file 2)", len(ids))): + print("# %s: %d (%.2f%%)" % (text, n, 100. * n / max(1, len(common)))) From 4022b5ae1929758312b5f7cd8bbee62e3e0adf80 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 12:38:53 +0100 Subject: [PATCH 32/60] GPU TPC: Continuous and less strict (during refit) NCl cut for given Pt --- GPU/GPUTracking/Base/GPUParam.h | 4 ++-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 7 ++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 2 +- .../SectorTracker/GPUTPCTrackletConstructor.cxx | 4 ++-- GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx | 2 +- 8 files changed, 15 insertions(+), 8 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUParam.h b/GPU/GPUTracking/Base/GPUParam.h index 30cc4d1e7dca9..5fab4b5e0fb54 100644 --- a/GPU/GPUTracking/Base/GPUParam.h +++ b/GPU/GPUTracking/Base/GPUParam.h @@ -80,8 +80,8 @@ struct GPUParam : public internal::GPUParam_t void UpdateRun3ClusterErrors(const float* yErrorParam, const float* zErrorParam); #endif - GPUd() constexpr uint32_t tpcMinHitsB5(float qPtB5) const { return CAMath::Abs(qPtB5) > 10 ? 10 : (CAMath::Abs(qPtB5) > 5 ? 15 : 29); } // Minimum hits should depend on Pt, low Pt tracks can have few hits. 29 Hits default, 15 for < 200 mev, 10 for < 100 mev - GPUd() constexpr float Alpha(int32_t iSector) const // TODO: Check if this is faster, or the lookup from GPUTPCGeometry + GPUd() constexpr uint32_t tpcMinHitsB5Abs(float qPtB5) const { return qPtB5 > 10.f ? 10 : (qPtB5 > 5.f ? (uint32_t)(20 - qPtB5) : (qPtB5 > 3.6f ? (uint32_t)(65 - 10.f * qPtB5) : 29)); } + GPUd() constexpr float Alpha(int32_t iSector) const // TODO: Check if this is faster, or the lookup from GPUTPCGeometry { if (iSector >= (int32_t)o2::tpc::constants::MAXSECTOR / 2) { iSector -= o2::tpc::constants::MAXSECTOR / 2; diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b8c25d4a87931..bf7d8b0f3017c 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -102,6 +102,7 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(trackFitMinHitsMaxQPtIncrease, float, 0.2f, "", 0, "For the QPt-dependent minimum number of hits of the track fit, abs(QPt) is increased by 1 sigma, but at most by this fraction (0 = use fitted QPt)") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 5b40560d9b61a..26ffff549bad2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1701,7 +1701,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread } nHits += nTrackHits; } - if (nHits < Param().tpcMinHitsB5(trbase->QPt() * Param().qptB5Scaler)) { + if (nHits < Param().tpcMinHitsB5Abs(CAMath::Abs(trbase->QPt() * Param().qptB5Scaler))) { break; } @@ -2163,6 +2163,11 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, #endif*/ } +GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const +{ + return Param().tpcMinHitsB5Abs((CAMath::Abs(qPt) + CAMath::Min(CAMath::Sqrt(CAMath::Max(0.f, err2QPt)), Param().rec.tpc.trackFitMinHitsMaxQPtIncrease * CAMath::Abs(qPt))) * Param().qptB5Scaler); +} + GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 774634f34bd1a..1ca312165f17a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -215,6 +215,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() uint32_t MinHitsForQPt(float qPt, float err2QPt) const; // QPt-dependent minimum number of hits, for QPt increased by 1 sigma (at most by the fraction trackFitMinHitsMaxQPtIncrease) GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index 4bcffd8a31c46..ddd6862b96bbb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -76,7 +76,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlock if (nCl == 0) { continue; } - if (nCl + 2 < merger.Param().tpcMinHitsB5(tracks[i].GetParam().GetQPt() * merger.Param().qptB5Scaler)) { // Give 2 hits tolerance in the primary leg, compared to the full fit of the looper + if (nCl + 2 < merger.MinHitsForQPt(tracks[i].GetParam().GetQPt(), tracks[i].GetParam().GetErr2QPt())) { // Give 2 hits tolerance in the primary leg, compared to the full fit of the looper continue; } if (merger.Param().rec.tpc.minNClustersFinalTrack != -1 && nCl < (uint32_t)merger.Param().rec.tpc.minNClustersFinalTrack) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c3d254a0b9cc2..a8a71f124ab49 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -447,7 +447,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ o2::utils::DebugStreamer::instance()->getStreamer("debug_accept_track", "UPDATE") << o2::utils::DebugStreamer::instance()->getUniqueTreeName("debug_accept_track").data() << "iTrk=" << iTrk << "outerParam=" << track.OuterParam() << "track=" << this << "ihitStart=" << ihitStart << "\n"; }) - if (!(N + NTolerated >= (int32_t)param.tpcMinHitsB5(mP[4] * param.qptB5Scaler) && 2 * NTolerated <= CAMath::Max(10, N) && CheckNumericalQuality(covYYUpd))) { + if (!(N + NTolerated >= (int32_t)merger.MinHitsForQPt(mP[4], mC[14]) && 2 * NTolerated <= CAMath::Max(10, N) && CheckNumericalQuality(covYYUpd))) { return false; // TODO: NTolerated should never become that large, check what is going wrong! } if (param.rec.tpc.minNClustersFinalTrack != -1 && N + NTolerated < param.rec.tpc.minNClustersFinalTrack) { diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 86979c29006ae..2f89a998f8e9f 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -56,8 +56,8 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_ { // reconstruction of tracklets, tracklet store step const uint32_t nHits = r.mLastRow + 1 - r.mFirstRow; - if (nHits == 0 || r.mNHits == 0 || (r.mNHits < tracker.Param().tpcMinHitsB5(tParam.QPt() * tracker.Param().qptB5Scaler) || !CheckCov(tParam) || CAMath::Abs(tParam.GetQPt() * tracker.Param().qptB5Scaler) > tracker.Param().rec.maxTrackQPtB5)) { - CADEBUG(printf(" Rejected: nHits %d QPt %f MinHits %d MaxQPt %f CheckCov %d\n", r.mNHits, tParam.QPt(), tracker.Param().tpcMinHitsB5(tParam.QPt() * tracker.Param().qptB5Scaler), tracker.Param().rec.maxTrackQPtB5, (int32_t)CheckCov(tParam))); + if (nHits == 0 || r.mNHits == 0 || (r.mNHits < tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tParam.QPt() * tracker.Param().qptB5Scaler)) || !CheckCov(tParam) || CAMath::Abs(tParam.GetQPt() * tracker.Param().qptB5Scaler) > tracker.Param().rec.maxTrackQPtB5)) { + CADEBUG(printf(" Rejected: nHits %d QPt %f MinHits %d MaxQPt %f CheckCov %d\n", r.mNHits, tParam.QPt(), tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tParam.QPt() * tracker.Param().qptB5Scaler)), tracker.Param().rec.maxTrackQPtB5, (int32_t)CheckCov(tParam))); return; } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx index 1f85718de6fa5..0a68a64c9a204 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx @@ -51,7 +51,7 @@ GPUdii() void GPUTPCTrackletSelector::Thread<0>(int32_t nBlocks, int32_t nThread uint32_t gap = 0; uint32_t nShared = 0; uint32_t nHits = 0; - const uint32_t minHits = tracker.Param().rec.tpc.minNClustersTrackSeed == -1 ? tracker.Param().tpcMinHitsB5(tracklet.Param().QPt() * tracker.Param().qptB5Scaler) : tracker.Param().rec.tpc.minNClustersTrackSeed; + const uint32_t minHits = tracker.Param().rec.tpc.minNClustersTrackSeed == -1 ? tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tracklet.Param().QPt() * tracker.Param().qptB5Scaler)) : tracker.Param().rec.tpc.minNClustersTrackSeed; const uint32_t sharingMinNorm = minHits * tracker.Param().rec.tpc.trackletMinSharedNormFactor; const float maxSharedNorm = maxSharedFrac * sharingMinNorm; From 728710822a63c0f2d03bf75d68d7e53d999719d8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 13:16:41 +0100 Subject: [PATCH 33/60] GPU GPUCA_CADEBUG: add possibility to have a condition for the debug output --- GPU/GPUTracking/Definitions/GPUDef.h | 22 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 ++--- .../Standalone/scripts/compareTrackNcl.py | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index 54c180d5f7187..d3aa2acfa51d5 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -70,10 +70,23 @@ #ifdef CADEBUG #undef CADEBUG #endif - #if GPUCA_CADEBUG == 1 && !defined(GPUCA_GPUCODE) - #define CADEBUG(...) __VA_ARGS__ - #define CADEBUG2(cmd, ...) {__VA_ARGS__; cmd;} - #define GPUCA_CADEBUG_ENABLED + #ifdef CADEBUG2 + #undef CADEBUG2 + #endif + #ifdef CADEBUG3 + #undef CADEBUG3 + #endif + #if GPUCA_CADEBUG >= 1 && !defined(GPUCA_GPUCODE) + #if GPUCA_CADEBUG >= 2 + #define CADEBUG(...) if (CADEBUG_CONDITION) {__VA_ARGS__;} + #define CADEBUG2(cmd, ...) if (CADEBUG_CONDITION) {__VA_ARGS__; cmd;} + #define GPUCA_CADEBUG_ENABLED 2 + #else + #define CADEBUG(...) __VA_ARGS__ + #define CADEBUG2(cmd, ...) {__VA_ARGS__; cmd;} + #define GPUCA_CADEBUG_ENABLED 1 + #endif + #define CADEBUG3(...) __VA_ARGS__ #endif #undef GPUCA_CADEBUG #endif @@ -81,5 +94,6 @@ #ifndef CADEBUG #define CADEBUG(...) #define CADEBUG2(cmd, ...) {cmd;} + #define CADEBUG3(...) #endif // clang-format on diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 26ffff549bad2..f0183cd5472b6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -17,6 +17,7 @@ // #define GPUCA_CADEBUG_ENABLED #include "GPUCommonDef.h" +#include "GPUDef.h" #if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) #include "AliHLTTPCClusterMCData.h" diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a8a71f124ab49..83cce0f9c4bfe 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -13,7 +13,6 @@ /// \author David Rohr, Sergey Gorbunov #define GPUCA_CADEBUG 0 -#define DEBUG_SINGLE_TRACK -1 // #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" @@ -41,6 +40,7 @@ #include #include #include "AliHLTTPCClusterMCData.h" +static thread_local bool CADEBUG_CONDITION = true; #endif #ifndef GPUCA_GPUCODE @@ -873,7 +873,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG3(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { @@ -1338,7 +1338,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - CADEBUG(if (DEBUG_SINGLE_TRACK != -1) { static const std::set dbgTracks = []() { std::set m; if (const char* e = DEBUG_SINGLE_TRACK == -2 ? getenv("DEBUG_TRACK") : nullptr) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } else { m.insert(DEBUG_SINGLE_TRACK); } return m; }(); if (!dbgTracks.count(iTrk)) { track.SetNClusters(0); track.SetOK(0); return; } }); + CADEBUG3(if (GPUCA_CADEBUG_ENABLED >= 2) { static const std::set dbgTracks = []() { std::set m; if (const char* e = getenv("DEBUG_TRACK")) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } return m; }(); CADEBUG_CONDITION = dbgTracks.count(iTrk); }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py index 01ff969cf17a6..03d863dbb55ef 100755 --- a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py +++ b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py @@ -11,7 +11,7 @@ # or submit itself to any jurisdiction. # Usage: compareTrackNcl.py FILE1 FILE2, with files of lines "Track NCl " (from --DEBUGmergerDumpTrackNCl). -# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with DEBUG_SINGLE_TRACK -2 in GPUTPCGMTrackParam.cxx, followed by some statistics. +# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with GPUCA_CADEBUG 2 in GPUTPCGMTrackParam.cxx, followed by some statistics. import sys From 3326a9cc67894ca7d985b0b3dbeb7b0af4d715a8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 17:11:57 +0100 Subject: [PATCH 34/60] GPU TPC: Make extrapolation more robust by stopping early when there is no chance of continuing --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index bf7d8b0f3017c..88eebb205f14b 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -106,6 +106,8 @@ AddOptionRTC(trackFitMinHitsMaxQPtIncrease, float, 0.2f, "", 0, "For the QPt-dep AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") +AddOptionRTC(maxSinPhiForFitUpdate, float, 0.9396926f, "", 0, "Max abs(sinPhi) of the track to update it with a cluster in the track fit)") // default sin(70 deg = 0.9396926 +AddOptionRTC(rebuildTrackMaxQPtDrift, float, 0.5f, "", 0, "Stop the track extrapolation during rebuild when QPt changed by more than this fraction of max(1, abs(QPt)) compared to before the extrapolation (0 = disabled)") AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") AddOptionRTC(rebuildTrackFallbackErr, float, 1.f, "", 0, "Error (distance^2) assigned to the original clusters of a track used as fallback candidates in rows that could not be searched during rebuild") AddOptionRTC(rebuildTrackFallbackTrackChi2, float, 4.f, "", 0, "Chi2 per cluster assumed for the hit weight of tracks without successful fit (NDF <= 0) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 83cce0f9c4bfe..808f18fe53858 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -59,9 +59,9 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); + const float maxSinForUpdate = param.rec.tpc.maxSinPhiForFitUpdate; GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; @@ -302,6 +302,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit float segmentStartChi2 = mChi2; + const float qPtStart = mP[4]; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers @@ -317,7 +318,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ fail = true; break; } - if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + if (CAMath::Abs(mP[2]) > maxSinForUpdate || CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate || mC[0] >= param.rec.tpc.rebuildTrackMaxNonIntCov || mC[2] >= param.rec.tpc.rebuildTrackMaxNonIntCov) { // No further clusters can be attached to the fit fail = true; break; } @@ -380,6 +381,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else + candidate.best = -1; + break; + } if (retValUpd != 0) { candidate.best = -1; if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { @@ -426,9 +431,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ candidate.best = -1; } } - } - - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped + StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); + } + } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } From 77cb61b12d47bace7b1c0873e53b2a59eede6a1a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 17:47:22 +0100 Subject: [PATCH 35/60] GPU TPC: revert state after extrapolation in case of problems --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 88eebb205f14b..2530dcaad5e87 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -183,6 +183,7 @@ AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 20, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 8, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMaxRowsBack, uint8_t, 3, "", 0, "After the track extrapolation, propagate the track back to the last accepted cluster if it is at most this number of rows away, otherwise revert to the track before the extrapolation") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 808f18fe53858..a2d16eead1db6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -302,7 +302,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit float segmentStartChi2 = mChi2; - const float qPtStart = mP[4]; + const float qPtStart = mP[4], alphaStart = prop.GetAlpha(); + const GPUTPCGMTrackParam paramStart = *this; // To revert to if the extrapolation fails + float lastFoundX = 0.f, lastFoundAlpha = 0.f; + bool revert = false; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers @@ -310,12 +313,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iExtrapolatedRow), tmpY, tmpZ)) { - fail = true; + fail = revert = true; break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iExtrapolatedRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { - fail = true; + fail = revert = true; break; } if (CAMath::Abs(mP[2]) > maxSinForUpdate || CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate || mC[0] >= param.rec.tpc.rebuildTrackMaxNonIntCov || mC[2] >= param.rec.tpc.rebuildTrackMaxNonIntCov) { // No further clusters can be attached to the fit @@ -338,7 +341,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } if (prop.RotateToAlpha(param.Alpha(sector))) { // GetPropagatedYZ / InverseTransformYZtoX of the next attempt need Y in the frame of the new sector - fail = true; + fail = revert = true; break; } } @@ -365,6 +368,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { candidate.best = -1; + revert = true; break; } const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? @@ -383,11 +387,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else candidate.best = -1; + revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { candidate.best = -1; if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { + revert = true; break; } found = false; // Cluster rejected by chi2, handle as missing row @@ -397,6 +403,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (found) { lastFoundRow = iExtrapolatedRow; + lastFoundX = mX; + lastFoundAlpha = prop.GetAlpha(); if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { if (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows) { lastGoodRow = iExtrapolatedRow; // TODO: Could optionally store the track state here, and revert to the good state in case we reject further clusters @@ -435,6 +443,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } + const int32_t rowsMoved = CAMath::Abs((int32_t)(iExtrapolatedRow < GPUTPCGeometry::NROWS ? iExtrapolatedRow : iExtrapolatedRow - up) - (int32_t)lastGoodRow); + if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == nWays - 3 && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it + *this = paramStart; + prop.SetTrack(this, alphaStart); + } } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); From d57cd93d24b518cebc376433d77c43271a9b70d1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 21:50:00 +0100 Subject: [PATCH 36/60] GPU TPC: Fix some cases of fit failues after track rebuild (bad initial state, better handling of high inclination flags, fallback clusters for short tracks) --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 2530dcaad5e87..6de5f8c9cc5f2 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -186,6 +186,8 @@ AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum AddOptionRTC(rebuildTrackExtrMaxRowsBack, uint8_t, 3, "", 0, "After the track extrapolation, propagate the track back to the last accepted cluster if it is at most this number of rows away, otherwise revert to the track before the extrapolation") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") +AddOptionRTC(rebuildTrackMaxRowsFromEnd, uint8_t, 10, "", 0, "If the last fitted hit of the outward way of the rebuild is more than this number of rows away from the end of the track, the final fit starts from the original track parameters") +AddOptionRTC(rebuildTrackFallbackMaxNCl, uint8_t, 20, "", 0, "For tracks with at most this number of clusters, keep the original cluster of a row as candidate during rebuild if no candidate was found but it passes the interpolated chi2 cut (0 = disabled)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a2d16eead1db6..2b9831f3a2879 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -244,8 +244,19 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } else { + if (!markHighIncl && nWays >= 2 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination + float err2Y, err2Z; + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, -1.f, 0.f, 0.f); + prop.Model().Y() = mP[0] = yy; // Like an update that takes the position fully from the cluster and leaves the other parameters untouched + prop.Model().Z() = mP[1] = zz; + mC[0] = err2Y; + mC[2] = err2Z; + mC[1] = mC[3] = mC[4] = mC[6] = mC[7] = mC[10] = mC[11] = 0.f; + lastUpdateX = mX; + } else { + nMissed2++; + } MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; CADEBUG(printf(" --- break-sinphi\n")); NTolerated++; if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { @@ -267,8 +278,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { - CADEBUG(printf("Reinit linearization\n")); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { // TODO: Could be investigated to reinit the linearization also when sinPhi drifts away from it (a reinit when QPt differs makes things worse, early in the way the QPt of the linearization is usually the better one) + CADEBUG(printf("Reinit linearization: way %d, QPt %7.2f (%7.2f), SP %5.2f (%5.2f), NDF %d\n", iWay, mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mNDF)); prop.SetTrack(this, prop.GetAlpha()); resetT0 = 20; } @@ -277,6 +288,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; + if (param.rec.tpc.rebuildTrackInFit && !rebuilt) { // The high inclination row range is valid only if it lasts until the end of the way, reset it since this later hit was fitted + auto& highInclRow = ((clusters[0].row > clusters[maxN - 1].row) ^ (iWay & 1)) ? merger.TrackRebuildHelper()[iTrk].highInclRowLow : merger.TrackRebuildHelper()[iTrk].highInclRowHigh; + if (highInclRow != 255) { + highInclRow = 255; + } + } } else if (retValHit == 1) { if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -457,6 +474,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ Alpha = prop.GetAlpha(); if (ihitStart != 0) { MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + if (CAMath::Abs(clusters[ihitStart].row - clusters[0].row) > param.rec.tpc.rebuildTrackMaxRowsFromEnd) { // The fit ended far away from the end of the track, the final fit cannot start from there, so let it start from the original parameters like the first way + const int32_t ndf = mNDF; + const float chi2 = mChi2; + *this = track.GetParam(); + mNDF = ndf; + mChi2 = chi2; + Alpha = track.GetAlpha(); + } } return true; } @@ -568,6 +593,12 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else { rejectChi2 = true; } + } else if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation + candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { From ebdf1091e03f2f064cf613fb4d32787d3773dc49 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 22:45:47 +0100 Subject: [PATCH 37/60] GPU TPC: Fix deltaTimeFwd/deltaTimeBwd after track rebuild --- GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 8 ++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 45 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 1 + 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index ddd6862b96bbb..b2009aba09d41 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -215,9 +215,11 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks if (track.PrevSegment() >= 0) { const GPUTPCGMMergedTrack* chkTrk = track.GetFirstSegment(tracks, merger.Param().rec.enableCyclicGraphWorkarounds); - const auto& firstPrevCluster = trackClusters[chkTrk->FirstClusterRef()]; - t1 = clusters->clustersLinear[firstPrevCluster.num].getTime(); - sector1 = firstPrevCluster.sector; + if (chkTrk->NClusters()) { // First segment can have lost all its clusters in the rebuild, then its cluster reference is invalid + const auto& firstPrevCluster = trackClusters[chkTrk->FirstClusterRef()]; + t1 = clusters->clustersLinear[firstPrevCluster.num].getTime(); + sector1 = firstPrevCluster.sector; + } } bool cce = track.CCE() && ((sector1 < MAXSECTOR / 2) ^ (sector2 < MAXSECTOR / 2)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2b9831f3a2879..234f20d8a2c7d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -81,6 +81,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; + if (rebuilt && maxN > 0 && param.par.continuousTracking && !track.CCE() && !track.Looper()) { // The rebuilt track can have other first / last clusters than the track the Z shift was computed for + const float t0 = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[0].num].getTime(), tN = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[maxN - 1].num].getTime(); + ConstrainZ(merger, clusters[0].sector, CAMath::Max(t0, tN), CAMath::Min(t0, tN)); + } int32_t ihitStart = 0; int32_t interpolatedStart = 0; float covYYUpd = 0.f; @@ -1327,28 +1331,29 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merg merger.GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } - { - float deltaT = merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + mTOffset += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + return -ConstrainZ(merger, sector, cltmax, cltmin, deltaZ); +} + +GPUd() float GPUTPCGMTrackParam::ConstrainZ(const GPUTPCGMMerger& GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float deltaZ) +{ + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); + // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); + float deltaT = 0.f; + if (mTOffset < minT) { + deltaT = minT - mTOffset; + } + if (mTOffset + deltaT > maxT) { + deltaT = maxT - mTOffset; + } + if (deltaT != 0.f) { + deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; - const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); - const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); - // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); - deltaT = 0.f; - if (mTOffset < minT) { - deltaT = minT - mTOffset; - } - if (mTOffset + deltaT > maxT) { - deltaT = maxT - mTOffset; - } - if (deltaT != 0.f) { - deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); - // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); - mTOffset += deltaT; - } - mP[1] -= deltaZ; } - // printf("\n"); - return -deltaZ; + mP[1] -= deltaZ; + return deltaZ; } GPUd() bool GPUTPCGMTrackParam::CheckCov() const diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 4619db960981d..1d98920cfb094 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -203,6 +203,7 @@ class GPUTPCGMTrackParam GPUd() void Rotate(float alpha); GPUd() float ShiftZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float clx); GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); + GPUd() float ConstrainZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float deltaZ = 0.f); // Shift Z / TOffset such that the clusters are within the TPC drift volume, applies in addition the pending shift deltaZ to Z, returns the total shift GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From c58a7f53c10fd1cb9c917d929c108cc3e682b738 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 13 Nov 2026 17:24:49 +0100 Subject: [PATCH 38/60] GPU TPC: Fix storing OuterParam after implementing track rebuild --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 234f20d8a2c7d..2852a52b637ac 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -96,15 +96,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; - if (iWay && (iWay & 1) == 0) { - StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! - } - int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; + if (iWay && finalFit) { + StoreOuter(&track.OuterParam(), rebuilt ? Alpha : prop.GetAlpha()); + } + ResetCovariance(); prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed From 69e9d8f629635dc9086b6953b2570d53cc31a4ae Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 13 Nov 2026 23:42:06 +0100 Subject: [PATCH 39/60] GPU TPC: Option to run interpolation-rejection after track rebuild --- .../reconstruction/test/testGPUCATracking.cxx | 1 - GPU/GPUTracking/Base/GPUParam.cxx | 1 + GPU/GPUTracking/Definitions/GPUSettingsList.h | 11 ++- GPU/GPUTracking/Global/GPUChainTracking.cxx | 12 --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 92 ++++++++++--------- 5 files changed, 55 insertions(+), 62 deletions(-) diff --git a/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx b/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx index 9d44e5c8b5890..f93f43e1cd258 100644 --- a/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx +++ b/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx @@ -64,7 +64,6 @@ BOOST_AUTO_TEST_CASE(CATracking_test1) config.configGRP.solenoidBzNominalGPU = solenoidBz; config.configGRP.grpContinuousMaxTimeBin = continuous ? GPUSettings::TPC_MAX_TF_TIME_BIN : 0; // Number of timebins in timeframe if continuous, 0 otherwise - config.configReconstruction.tpc.nWays = 3; // Should always be 3! config.configReconstruction.tpc.searchWindowDZDR = 2.5f; // Should always be 2.5 for looper-finding and/or continuous tracking config.configReconstruction.tpc.trackReferenceX = refX; diff --git a/GPU/GPUTracking/Base/GPUParam.cxx b/GPU/GPUTracking/Base/GPUParam.cxx index 6650248439626..b7f8ccb9fa668 100644 --- a/GPU/GPUTracking/Base/GPUParam.cxx +++ b/GPU/GPUTracking/Base/GPUParam.cxx @@ -113,6 +113,7 @@ void GPUParam::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessi if (d) { rec.dyn = *d; } + rec.tpc.nWays = 1 + 2 * ((rec.tpc.mergerInitial3wayFit > 0) + (rec.tpc.mergerInterpolateErrors > 0) + (rec.tpc.rebuildTrackInFit > 0)); } void GPUParam::UpdateBzOnly(float newSolenoidBz, bool assumeConstantBz) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 6de5f8c9cc5f2..7c9a6a23ae614 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -148,8 +148,11 @@ AddOptionRTC(cfMinSplitNum, uint8_t, 1, "", 0, "Minimum number of split charges AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression") AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") -AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") -AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(mergerInitial3wayFit, uint8_t, 0, "", 0, "The track fit does 2 initial fit passes for a good track state before running the rebuild or the interpolation rejection, ads 2 fit iteration") +AddOptionRTC(mergerInterpolateErrors, uint8_t, 0, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection, adds 2 fit iterations") +AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") +AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions, adds 2 fit iterations") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history @@ -166,9 +169,6 @@ AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") -AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") -AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") -AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") @@ -190,6 +190,7 @@ AddOptionRTC(rebuildTrackMaxRowsFromEnd, uint8_t, 10, "", 0, "If the last fitted AddOptionRTC(rebuildTrackFallbackMaxNCl, uint8_t, 20, "", 0, "For tracks with at most this number of clusters, keep the original cluster of a row as candidate during rebuild if no candidate was found but it passes the interpolated chi2 cut (0 = disabled)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC +AddVariableRTC(nWays, uint8_t, 0) AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index df74187a4f9d5..2761fcb3ec733 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -258,18 +258,6 @@ bool GPUChainTracking::ValidateSteps() bool GPUChainTracking::ValidateSettings() { int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCCompression).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode; - if ((param().rec.tpc.nWays & 1) == 0) { - GPUError("nWay setting musst be odd number!"); - return false; - } - if (param().rec.tpc.mergerInterpolateErrors && param().rec.tpc.nWays < 3) { - GPUError("Cannot do error interpolation with NWays < 3!"); - return false; - } - if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { - GPUError("Need error interpolation to rebuild tracks during fit"); - return false; - } if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); return false; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2852a52b637ac..97296e6808cbf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -70,13 +70,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + if ((param.rec.tpc.rebuildTrackInFit && !rebuilt) || param.rec.tpc.mergerInterpolateErrors) { // Both the rebuild ways and the interpolation-rejection ways fill the interpolation + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range interpolation.hit[i].markInvalid(); } - if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; - } + } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } const int32_t nWays = param.rec.tpc.nWays; @@ -89,20 +89,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; + const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; - // TODO: Could do 5 ways and interpolate again on the rebuilt clusters and reject in the final way based on that (and get the outer param from the rebuilt clusters), but costs 2 more ways - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); - const bool finalOutInFit = iWay + 2 >= nWays; + const bool finalOutInFit = iWay >= nWays - 2; const bool finalFit = iWay == nWays - 1; if (iWay && finalFit) { - StoreOuter(&track.OuterParam(), rebuilt ? Alpha : prop.GetAlpha()); + StoreOuter(&track.OuterParam(), rebuilt && iWay == wayRebuildStart + 2 ? Alpha : prop.GetAlpha()); // In the first way of the rebuilt pass, prop is not yet initialized } ResetCovariance(); @@ -110,7 +110,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && (!rebuilt || iWay > wayRebuildStart + 2) ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -141,7 +141,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, (param.rec.tpc.rebuildTrackInFit && !param.rec.tpc.mergerInterpolateErrors) ? rebuilt : allowChangeClusters); // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; @@ -149,7 +149,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); - if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + if (rebuilt) { + interpolationIndex = ihit; + } else if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); fitBroken = true; break; @@ -161,7 +163,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides - const uint8_t doInterpolate = ((param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); + const uint8_t doInterpolate = (!param.rec.tpc.rebuildTrackInFit || (param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : (iWay == wayRebuildStart ? 1 : (iWay == wayRebuildStart + 1 ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -181,7 +183,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } - interpolationIndex += rowGap * wayDirection; + if (!rebuilt) { + interpolationIndex += rowGap * wayDirection; + } } lastPropagateRow = cluster.row; @@ -195,7 +199,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); } continue; @@ -205,7 +209,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } fitBroken = true; @@ -214,7 +218,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + if (iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { @@ -231,7 +235,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ HandleCrossCE(param, cluster.sector, lastSector); lastSector = cluster.sector; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + if ((param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) || (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart)) { prop.InterpolateFill(&inter); } @@ -248,7 +252,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } else { - if (!markHighIncl && nWays >= 2 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination + if (!markHighIncl && nWays > 1 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination float err2Y, err2Z; prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, -1.f, 0.f, 0.f); prop.Model().Y() = mP[0] = yy; // Like an update that takes the position fully from the cluster and leaves the other parameters untouched @@ -299,7 +303,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } fitBroken = true; @@ -307,7 +311,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); + assert(((!param.rec.tpc.rebuildTrackInFit || iWay != wayRebuildStart + 1) && (!param.rec.tpc.mergerInterpolateErrors || iWay != nWays - 2)) || ihit || interpolationIndex == 0); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -315,7 +319,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= wayRebuildStart && !(fitBroken && iWay == wayRebuildStart + 1) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); @@ -465,7 +469,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } const int32_t rowsMoved = CAMath::Abs((int32_t)(iExtrapolatedRow < GPUTPCGeometry::NROWS ? iExtrapolatedRow : iExtrapolatedRow - up) - (int32_t)lastGoodRow); - if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == nWays - 3 && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it + if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == wayRebuildStart && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it *this = paramStart; prop.SetTrack(this, alphaStart); } @@ -474,7 +478,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); @@ -567,14 +571,16 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const bool finalOutInFit = iWay >= nWays - 2; const auto& cluster = clusters[ihit]; int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - if (param.rec.tpc.rebuildTrackInFit) { + if (param.rec.tpc.rebuildTrackInFit && (finalOutInFit || iWay < wayRebuildStart + 2)) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds } } else { @@ -589,27 +595,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (!retryAttempt) { - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { - rejectChi2 = true; - } - } else if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { - const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation - candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; - } - } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; + if (finalFit && (param.rec.tpc.mergerInterpolateErrors || param.rec.tpc.rebuildTrackInFit)) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } + } else if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { + if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation + candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; } } - } else { + } else if (!param.rec.tpc.mergerInterpolateErrors && !param.rec.tpc.rebuildTrackInFit) { rejectChi2 = allowChangeClusters; } } From c032ed742a6955d9ccdf78a8f4eb90827363e60f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 15 Nov 2026 14:18:02 +0100 Subject: [PATCH 40/60] GPU TPC: Some fixes for NTolerated and nMissed, and some cleanup --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 10 ++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 40 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 7c9a6a23ae614..b7e96fba76cbc 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -134,7 +134,9 @@ AddOptionRTC(mergerLooperQPtB5Limit, uint8_t, 4, "", 0, "Min Q/Pt (@B=0.5T) to r AddOptionRTC(mergerLooperSecondHorizontalQPtB5Limit, uint8_t, 2, "", 0, "Min Q/Pt (@B=0.5T) to attempt second horizontal merge between sectors after a vertical merge was found") AddOptionRTC(trackFollowingMaxRowGap, uint8_t, 4, "", 0, "Maximum number of consecutive rows without hit in track following") AddOptionRTC(trackFollowingMaxRowGapSeed, uint8_t, 2, "", 0, "Maximum number of consecutive rows without hit in track following during fit of seed") -AddOptionRTC(trackFitMaxRowMissedHard, uint8_t, 10, "", 0, "Hard limit for number of missed rows in fit / propagation") +AddOptionRTC(trackFitAbortNMissedHits, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") +AddOptionRTC(trackFitAbortNMissedHitsHard, uint8_t, 10, "", 0, "Hard limit for number of missed rows in fit / propagation") +AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(extrapolationTrackingRowRange, uint8_t, 45, "", 0, "Number of rows from the upped/lower limit to search for extrapolated track candidates in for") AddOptionRTC(extrapolationTrackingMinRows, uint8_t, 10, "", 0, "Min num of rows an additional extrapolated track must span over") AddOptionRTC(extrapolationTrackingMinHits, uint8_t, 8, "", 0, "Min num of hits for an additional extrapolated track") @@ -156,8 +158,6 @@ AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely dur AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") -AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") @@ -170,7 +170,7 @@ AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for T AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without chi2 cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") @@ -500,7 +500,7 @@ AddShortcut("compare", 0, "--QAinput", "Compare QA histograms", "-c", "--qa", "- AddHelp("help", 'h') EndConfig() -//Settings for the O2 workflow +// Settings for the O2 workflow #if !defined(QCONFIG_PARSER_CXX) && !defined(GPUCA_STANDALONE) BeginSubConfig(GPUSettingsO2, global, configStandalone, "O2", 0, "O2 workflow settings", global) AddOption(solenoidBzNominalGPU, float, -1e6f, "", 0, "Field strength of solenoid Bz in kGaus") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 97296e6808cbf..d303d2baf8d54 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -92,8 +92,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? - int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! + int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); @@ -114,7 +114,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); - N = 0; + N = NTolerated = 0; uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; bool fitBroken = false; float lastUpdateX = -1; @@ -123,14 +123,18 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if ((param.rec.tpc.trackFitAbortNMissedHits > 0 && nMissed >= param.rec.tpc.trackFitAbortNMissedHits) || nMissedHard >= param.rec.tpc.trackFitAbortNMissedHitsHard) { // Too many missed hits in a row, abort this way and drop the remaining hits + CADEBUG(printf("\tAborting at hit %d: %d, %d hits rejected\n", ihit, nMissed, nMissedHard)); + if (finalOutInFit) { + MarkClusters(clusters, ihit, wayDirection > 0 ? maxN - 1 : 0, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + break; + } + if ((clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, flag %X\n", ihit, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { NTolerated++; } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; - } continue; } } @@ -146,7 +150,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const auto& cluster = clusters[ihit]; CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d, %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed, nMissedHard)); const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); if (rebuilt) { @@ -197,7 +201,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ retValProp) { // failed for other reason but rotation CADEBUG(printf(" --- break-prop\n")); MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; + nMissedHard++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); @@ -241,7 +245,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (currentClusterStatus) { nMissed++; - nMissed2++; + nMissedHard++; continue; } @@ -262,7 +266,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ mC[1] = mC[3] = mC[4] = mC[6] = mC[7] = mC[10] = mC[11] = 0.f; lastUpdateX = mX; } else { - nMissed2++; + nMissedHard++; } MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); CADEBUG(printf(" --- break-sinphi\n")); @@ -283,7 +287,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissedHard, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { // TODO: Could be investigated to reinit the linearization also when sinPhi drifts away from it (a reinit when QPt differs makes things worse, early in the way the QPt of the linearization is usually the better one) @@ -567,7 +571,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissedHard, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -636,7 +640,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - nMissed = nMissed2 = 0; + nMissed = nMissedHard = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track @@ -645,10 +649,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else if (finalFit) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } - if (!retValInt) { - nMissed++; - nMissed2++; - } + nMissed++; + nMissedHard++; return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 1d98920cfb094..96d37d2fd3fae 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,7 +156,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); From bd576589a87de34a620b32effbdb9c8126f42753 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 16 Nov 2026 22:51:44 +0100 Subject: [PATCH 41/60] GPU QA: Improve Efficiency Stat Printing --- GPU/GPUTracking/qa/GPUQA.cxx | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 2038c4c84f707..fd0fb18b6dfa3 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2355,10 +2355,31 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } if (mConfig.enableLocalOutput && !mConfig.inputHistogramsOnly && (mQATasks & taskTrackingEff) && mcPresent()) { - GPUInfo("QA Stats: Eff: Tracks Prim %d (Eta %d, Pt %d) %f%% (%f%%) Sec %d (Eta %d, Pt %d) %f%% (%f%%) - Res: Tracks %d (Eta %d, Pt %d)", (int32_t)mEff[3][1][0][0]->GetEntries(), (int32_t)mEff[3][1][0][3]->GetEntries(), (int32_t)mEff[3][1][0][4]->GetEntries(), - mEff[0][0][0][0]->GetSumOfWeights() / std::max(1., mEff[3][0][0][0]->GetSumOfWeights()), mEff[0][1][0][0]->GetSumOfWeights() / std::max(1., mEff[3][1][0][0]->GetSumOfWeights()), (int32_t)mEff[3][1][1][0]->GetEntries(), (int32_t)mEff[3][1][1][3]->GetEntries(), - (int32_t)mEff[3][1][1][4]->GetEntries(), mEff[0][0][1][0]->GetSumOfWeights() / std::max(1., mEff[3][0][1][0]->GetSumOfWeights()), mEff[0][1][1][0]->GetSumOfWeights() / std::max(1., mEff[3][1][1][0]->GetSumOfWeights()), (int32_t)mRes2[0][0]->GetEntries(), - (int32_t)mRes2[0][3]->GetEntries(), (int32_t)mRes2[0][4]->GetEntries()); + GPUInfo("QA Stats: Tracking performance vs Pt: average over the Pt bins (all tracks)"); + static constexpr const char* names[4] = {"Prim findable", "Prim all", "Sec findable", "Sec all"}; + for (int32_t j = 0; j < 4; j++) { + const int32_t iFindable = j % 2, iSec = j / 2; // index 0 of the findable dimension are the findable tracks, index 1 all tracks + double sumNum[3] = {}, sumDen[3] = {}, sumRatio[3] = {}; + int32_t nBins[3] = {}; + for (int32_t bin = 1; bin <= mEff[0][iFindable][iSec][4]->GetNbinsX(); bin++) { + const double rec = mEff[0][iFindable][iSec][4]->GetBinContent(bin), clone = mEff[1][iFindable][iSec][4]->GetBinContent(bin), fake = mEff[2][iFindable][iSec][4]->GetBinContent(bin); + const double num[3] = {rec, clone, fake}, den[3] = {mEff[5][iFindable][iSec][4]->GetBinContent(bin), rec + clone, rec + clone + fake}; // Same ratios as in the plots + for (int32_t l = 0; l < 3; l++) { + sumNum[l] += num[l]; + sumDen[l] += den[l]; + if (den[l] > 0.) { + sumRatio[l] += num[l] / den[l]; + nBins[l]++; + } + } + } + double avg[3], total[3]; + for (int32_t l = 0; l < 3; l++) { + avg[l] = nBins[l] ? 100. * sumRatio[l] / nBins[l] : 0.; + total[l] = sumDen[l] > 0. ? 100. * sumNum[l] / sumDen[l] : 0.; + } + GPUInfo("QA Stats: %-13s - Eff %5.2f (%6.2f) - Clone %5.2f (%6.2f) - Fake %5.2f (%6.2f)", names[j], avg[0], total[0], avg[1], total[1], avg[2], total[2]); + } } int32_t flagShowVsPtLog = (mConfig.enableLocalOutput || mConfig.shipToQCAsCanvas) ? 1 : 0; From d4e819d6af813b0e575c3d6f58f6a3d2d5e8a7e6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 16 Nov 2026 20:12:25 +0100 Subject: [PATCH 42/60] GPU TPC: Make AttachClusters work with track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 64 ----- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 1 - GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 218 +++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 16 +- 5 files changed, 165 insertions(+), 136 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b7e96fba76cbc..4fca6292cd698 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -157,7 +157,7 @@ AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF o AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions, adds 2 fit iterations") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") -AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: interpolate missing rows, 4: extrapolate inwards / outwards, 8: loop following)") // TODO: Add history AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index ddfefe8636af5..fcaa430bfdd94 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -652,70 +652,6 @@ GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::Interpolati } } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) -{ - float* GPUrestrict() mC = mT->Cov(); - float* GPUrestrict() mP = mT->Par(); - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } - return 0; -} - GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* GPUrestrict() param) { float* GPUrestrict() mC = mT->Cov(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index 3f1a74cfb6c26..9c31069da5302 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -103,7 +103,6 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index d303d2baf8d54..98593716d99c2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -90,6 +90,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; + const bool adjacentInRebuild = param.rec.tpc.rebuildTrackInFit && !param.rec.tpc.mergerInterpolateErrors; + const int8_t adjacentLeg = 1 + (track.Leg() == 0); for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. @@ -115,13 +117,18 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = NTolerated = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255, lastIndexRow = 255, lastIndexSector = 255; bool fitBroken = false; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (lastIndexRow != 255) { // One index per pad row (2 if the track has clusters of 2 sectors in the row), counted also for hits skipped below, such that all ways use the same index for a hit + interpolationIndex += (clusters[ihit].row == lastIndexRow ? (int32_t)(clusters[ihit].sector != lastIndexSector) : CAMath::Abs(clusters[ihit].row - lastIndexRow)) * wayDirection; + } + lastIndexRow = clusters[ihit].row; + lastIndexSector = clusters[ihit].sector; if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitAbortNMissedHits > 0 && nMissed >= param.rec.tpc.trackFitAbortNMissedHits) || nMissedHard >= param.rec.tpc.trackFitAbortNMissedHitsHard) { // Too many missed hits in a row, abort this way and drop the remaining hits CADEBUG(printf("\tAborting at hit %d: %d, %d hits rejected\n", ihit, nMissed, nMissedHard)); @@ -155,8 +162,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); if (rebuilt) { interpolationIndex = ihit; - } else if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic - merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); + } else if ((uint32_t)interpolationIndex >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); fitBroken = true; break; } @@ -164,15 +171,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (rowGap) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides + const bool doAttach = (adjacentInRebuild ? (iWay == wayRebuildStart + 1 && !ceOverlap) : (allowChangeClusters && !param.rec.tpc.rebuildTrackInFit)) && !(param.rec.tpc.disableMarkAdjacent & 2); const uint8_t doInterpolate = (!param.rec.tpc.rebuildTrackInFit || (param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : (iWay == wayRebuildStart ? 1 : (iWay == wayRebuildStart + 1 ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; - for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { + for (int32_t iRow = lastPropagateRow + step, index = rebuilt ? interpolationIndex : (interpolationIndex - rowGap * wayDirection); iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; @@ -187,9 +194,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } - if (!rebuilt) { - interpolationIndex += rowGap * wayDirection; - } } lastPropagateRow = cluster.row; @@ -204,7 +208,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ nMissedHard++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } continue; } @@ -214,22 +218,17 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } fitBroken = true; break; // bad chi2 for the whole track, stop the fit } float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps - } - if (allowChangeClusters) { - AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isValid()) : allowChangeClusters)) { + uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; @@ -308,7 +307,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } else if (retValHit == 1) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } fitBroken = true; break; @@ -381,7 +380,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); - float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, true); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; bool found = candidate.id >= 2; if (found) { @@ -468,7 +467,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ candidate.best = -1; } } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } @@ -477,7 +476,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ *this = paramStart; prop.SetTrack(this, alphaStart); } - } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -540,18 +539,18 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } } - if (doAttach) { - AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); - } - if (doInterpolate) { - if (doInterpolate == 2) { - if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { - inter.markInvalid(); - } - FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); - } else { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, doAttach ? (1 + (track.Leg() == 0)) : 0); + } else { + if (doInterpolate == 1) { prop.InterpolateFill(&inter); } + if (doAttach) { + FindAdjacentClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } } } @@ -605,14 +604,14 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } else if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + retValInt = InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, (param.rec.tpc.disableMarkAdjacent & 1) ? 0 : (1 + (track.Leg() == 0))); } else { rejectChi2 = true; } } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + if (candidate.id < 2 && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; } @@ -657,7 +656,82 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, bool extrapolation) +GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUrestrict() merger, const GPUTPCGMPropagator& GPUrestrict() prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + float chi2Y, chi2Z, ImP0, ImP1, ImC0, ImC2; // Interpolated position and errors + if (mNDF <= 0) { + ImP0 = (float)inter->posY; + ImP1 = (float)inter->posZ + deltaZ; + ImC0 = (float)inter->errorY; + ImC2 = (float)inter->errorZ; + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (prop.GetFitInProjections()) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (doAttach) { // Search adjacent clusters here, at the interpolated position with its errors (1 = other leg, 2 = good leg) + float uncorrectedY, uncorrectedZ, clErr2Y, clErr2Z; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) <= merger.GetConstantMem()->tpcTrackers[sector].Row(row).getTPCMaxY()) { + param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); + } + } + if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return GPUTPCGMPropagator::updateErrorClusterRejectedInInterpolation; + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -671,7 +745,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; } const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; - if (rowData.NHits() && (useInter || useCurrent)) { + if (useInter || useCurrent) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -726,6 +800,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { float err2Y, err2Z; param.GetClusterErrors2(sector, row, zMoved, sinPhiMoved, mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + if (doAttach) { + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, err2Y + CAMath::Abs(ImC0), err2Z + CAMath::Abs(ImC2)); + } const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(ImC0))); // TODO: is 4 a good factor?? @@ -794,11 +871,19 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } + } else if (doAttach && (!interOnly || inter.isValid())) { + const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, y, z, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, z, err2Y + (interOnly ? (float)inter.errorY : CAMath::Abs(mC[0])), err2Z + (interOnly ? (float)inter.errorZ : CAMath::Abs(mC[2]))); + } } return uncorrectedY; } -GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, const int8_t doAttach) { for (; ihit >= 0; ihit--, interpolationIndex--) { while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { @@ -813,7 +898,7 @@ GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrest return; } auto inter = interpolation.hit[interpolationIndex]; - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, doAttach); } } @@ -998,7 +1083,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; @@ -1007,22 +1092,38 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric Y = mP[0]; Z = mP[1]; } - return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); + return FindAdjacentClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } + + float uncorrectedY, uncorrectedZ; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { + return uncorrectedY; + } + + float err2Y, err2Z; + param.GetClusterErrors2(sector, iRow, Z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, iRow, iTrack, goodLeg, uncorrectedY, uncorrectedZ, Z, err2Y + CAMath::Abs(mC[0]), err2Z + CAMath::Abs(mC[2])); // Cov can be bogus when following circle + + return uncorrectedY; +} + +GPUd() void GPUTPCGMTrackParam::FindAdjacentClustersUncorrectedPos(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float uncorrectedY, float uncorrectedZ, float Z, float err2Y, float err2Z) +{ + const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() == 0) { - return -1e6f; - } const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); @@ -1031,15 +1132,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric const float stepZ = row.HstepZ(); int32_t bin, ny, nz; - float uncorrectedY, uncorrectedZ; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); - if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { - return uncorrectedY; - } - bool protect = CAMath::Abs(GetQPt() * param.qptB5Scaler) <= param.rec.tpc.rejectQPtB5 && goodLeg; - float err2Y, err2Z; - param.GetClusterErrors2(sector, iRow, Z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge const float tubeMaxSize2 = protect ? param.rec.tpc.tubeProtectMaxSize2 : param.rec.tpc.tubeRemoveMaxSize2; const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; @@ -1050,8 +1143,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { tubeSigma2 *= protect ? 2 : 0.5; } - const float sy2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * (err2Y + CAMath::Abs(mC[0])))); // Cov can be bogus when following circle - const float sz2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * (err2Z + CAMath::Abs(mC[2])))); // In that case we should provide the track error externally + const float sy2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * err2Y)); + const float sz2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * err2Z)); const float tubeY = CAMath::Sqrt(sy2); const float tubeZ = CAMath::Sqrt(sz2); const float sy21 = 1.f / sy2; @@ -1093,7 +1186,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric } } } - return uncorrectedY; } GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) @@ -1157,13 +1249,13 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { - data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); + data.param.FindAdjacentClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); } else { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.FindAdjacentClustersLooperMirror(merger, data.sector, data.row, data.track, data.outwards, prop); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) +GPUdi() void GPUTPCGMTrackParam::FindAdjacentClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -1205,7 +1297,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + FindAdjacentClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -1231,7 +1323,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::FindAdjacentClustersLooperMirror(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; @@ -1280,7 +1372,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + FindAdjacentClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 96d37d2fd3fae..338f3d7d8b910 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,13 +163,15 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, bool extrapolation = false); - GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); + GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg) + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); + + GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void FindAdjacentClustersUncorrectedPos(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float uncorrectedY, float uncorrectedZ, float Z, float err2Y, float err2Z); // err2Y / err2Z: cluster error + track error + GPUd() void FindAdjacentClustersLooperMirror(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void FindAdjacentClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 6651245c334ff8eddca9192d3176c541005813c1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 11:38:46 +0100 Subject: [PATCH 43/60] GPU TPC: Mark hits with findAdjacentClusters done in previous iteration to cover exactly the remaining in last fit iteration. --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 88388a4b9074f..ba6abedbcc1ad 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -37,6 +37,8 @@ struct InterpolationErrorHit { GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } + GPUdi() bool isAdjacentDone() const { return errorY == (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } + GPUdi() void markAdjacentDone() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Adjacent clusters were searched at the interpolated position of this entry, the entry is invalid afterwards }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 98593716d99c2..2a89e3814f824 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -227,7 +227,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps - } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isValid()) : allowChangeClusters)) { + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isAdjacentDone()) : allowChangeClusters)) { uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { @@ -723,6 +723,7 @@ GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUre param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); } + inter->markAdjacentDone(); } if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 338f3d7d8b910..2e8ee86ca8196 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,7 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg) + GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg), marks inter as done GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); From 1be019db1162593baa98981ada220759b5d3da7d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 11:33:18 +0100 Subject: [PATCH 44/60] GPU TPC: Fix cluster adjacent flags with Rebuild (and improve without) --- .../GPUTPCCompressionKernels.cxx | 4 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 20 +++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 4 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 9 ++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 2 +- .../display/render/GPUDisplayDraw.cxx | 2 +- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 9bc6f40a052fa..67dacf993f0a4 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -53,7 +53,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread= GPUSettings::RejectionStrategyA && !(attach & gputpcgmmergertypes::attachProtect) && (rejectTrk || GPUTPCClusterRejection::GetIsRejected(attach)); @@ -206,7 +206,7 @@ GPUd() bool GPUTPCCompression::rejectCluster(int32_t idx, const GPUParam& GPUres if (attach & gputpcgmmergertypes::attachProtect) { return false; } - int32_t id = attach & gputpcgmmergertypes::attachTrackMask; + int32_t id = gputpcgmmergertypes::getAttachedTrackId(attach); auto& trk = ioPtrs.mergedTracks[id]; if (GPUTPCClusterRejection::IsTrackRejected(trk, param)) { return true; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index f0183cd5472b6..31a6b9a18a10b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1878,13 +1878,10 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in mTrackOrderAttach[mTrackSort[i]] = i; GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { + const bool protect = CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0; for (uint32_t j = 0; j < trk.NClusters(); j++) { - if (!Param().rec.tpc.rebuildTrackInFit) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; - } - CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); + if (protect) { // Protect the clusters of the initial tracks. No track id on purpose, any entry of a final track for the cluster shall win + CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], (uint32_t)(attachAttached | attachProtect | (Param().rec.tpc.rebuildTrackInFit ? attachZero : attachGood))); } CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } @@ -1933,8 +1930,8 @@ GPUd() void GPUTPCGMMerger::Finalize0(int32_t nBlocks, int32_t nThreads, int32_t mTrackSort[mTrackOrderAttach[i]] = i; } for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nThreads * nBlocks) { - if (!(mClusterAttachment[mClusters[i].num] & attachProtect)) { - mClusterAttachment[mClusters[i].num] = 0; // Reset adjacent attachment for attached clusters, set correctly below + if (!(mClusterAttachment[mClusters[i].num] & attachProtect)) { // Protected entries are kept on purpose: resetting them could release clusters of the initial tracks that were used for the seeding, and removing such clusters can make the tracking not find the track again + mClusterAttachment[mClusters[i].num] = 0; // Reset adjacent attachment for attached clusters, set correctly below } } } @@ -1948,11 +1945,12 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t } for (uint32_t j = 0; j < trk.NClusters(); j++) { int32_t id = mClusters[trk.FirstClusterRef() + j].num; - uint32_t weight = mTrackOrderAttach[i] | attachAttached; + uint32_t weight = (mTrackOrderAttach[i] << nPureFlags) | attachAttached; uint8_t clusterState = mClusters[trk.FirstClusterRef() + j].state; if (!(clusterState & GPUTPCGMMergedTrackHit::flagReject)) { weight |= attachGood; - } else if (clusterState & GPUTPCGMMergedTrackHit::flagHighIncl) { + } + if (clusterState & GPUTPCGMMergedTrackHit::flagHighIncl) { weight |= attachHighIncl; } if (trk.Leg() == 0) { @@ -1970,7 +1968,7 @@ GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { - mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; + mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | (mTrackSort[getAttachedTrackId(mClusterAttachment[i])] << nPureFlags); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index d22fd2e05e0a0..2ce927a5a5f7d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -206,7 +206,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const if (j++ % 10 == 0) { out << "\n Cluster attachment "; } - out << i << ": " << (mClusterAttachment[i] & attachTrackMask) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; + out << i << ": " << getAttachedTrackId(mClusterAttachment[i]) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; } } out << "\n"; @@ -255,7 +255,7 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const if (++j % 10 == 0) { out << " Cluster attachment "; } - out << i << ": " << (mClusterAttachment[i] & attachTrackMask) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; + out << i << ": " << getAttachedTrackId(mClusterAttachment[i]) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; if (j % 10 == 0) { out << "\n"; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index ba6abedbcc1ad..7dd7d31a4f9cb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -27,11 +27,14 @@ enum attachTypes { attachProtect = 0x80000000, attachGood = 0x20000000, attachGoodLeg = 0x10000000, attachTube = 0x08000000, - attachHighIncl = 0x04000000, - attachTrackMask = 0x03FFFFFF, - attachFlagMask = 0xFC000000, + attachHighIncl = 0x1, + attachTrackMask = 0x07FFFFFE, + attachFlagMask = 0xF8000001, + nPureFlags = 1, // Number of flag bits below the track id (do not influence which track wins the attachment) attachZero = 0 }; +GPUdi() static constexpr uint32_t getAttachedTrackId(uint32_t v) { return (v & attachTrackMask) >> nPureFlags; } + struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2a89e3814f824..a8b13a7bfe480 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -1155,7 +1155,7 @@ GPUd() void GPUTPCGMTrackParam::FindAdjacentClustersUncorrectedPos(const GPUTPCG const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + uint32_t myWeight = (merger.TrackOrderAttach()[iTrack] << gputpcgmmergertypes::nPureFlags) | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; diff --git a/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx b/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx index 7e85a14674319..4af2af4f3147c 100644 --- a/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx +++ b/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx @@ -170,7 +170,7 @@ void GPUDisplay::DrawClusters(int32_t iSector) } else if ((mCfgH.markAdjacentClusters & 16) && (attach & gputpcgmmergertypes::attachHighIncl)) { select = tMARKED; } else if (mCfgH.markAdjacentClusters & 8) { - if (fabsf(mIOPtrs->mergedTracks[attach & gputpcgmmergertypes::attachTrackMask].GetParam().GetQPt()) > 20.f) { + if (fabsf(mIOPtrs->mergedTracks[gputpcgmmergertypes::getAttachedTrackId(attach)].GetParam().GetQPt()) > 20.f) { select = tMARKED; } } From 645a8787cd3f7e5c266609719d661d1f85e42c65 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 09:28:25 +0100 Subject: [PATCH 45/60] GPU QA: Add histograms for adjacent clusters without rejection, Fix some naming --- GPU/GPUTracking/qa/GPUQA.cxx | 83 +++++++++++++++++++++++++++--------- GPU/GPUTracking/qa/GPUQA.h | 18 ++++---- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index fd0fb18b6dfa3..13bafef79312d 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -100,11 +100,11 @@ inline checkClusterStateResult GPUQA::checkClusterState(uint32_t attach, T* coun { checkClusterStateResult r; r.unattached = attach == 0; - r.id = attach & gputpcgmmergertypes::attachTrackMask; + r.id = gputpcgmmergertypes::getAttachedTrackId(attach); if (!r.unattached && !(attach & gputpcgmmergertypes::attachProtect)) { r.qpt = fabsf(mTracking->mIOPtrs.mergedTracks[r.id].GetParam().GetQPt()); r.lowPt = r.qpt * mTracking->GetParam().qptB5Scaler > mTracking->GetParam().rec.tpc.rejectQPtB5; - r.mev200 = r.qpt > 5; + r.mev200 = r.qpt * mTracking->GetParam().qptB5Scaler > 5; r.mergedLooperUnconnected = mTracking->mIOPtrs.mergedTracks[r.id].MergedLooperUnconnected(); r.mergedLooperConnected = mTracking->mIOPtrs.mergedTracks[r.id].MergedLooperConnected(); } @@ -172,9 +172,9 @@ static const constexpr char* AXIS_TITLES[5] = {"#it{y}-#it{y}_{mc} (mm) (Resolut static const constexpr char* AXIS_TITLES_NATIVE[5] = {"#it{y}-#it{y}_{mc} (mm) (Resolution)", "#it{z}-#it{z}_{mc} (mm) (Resolution)", "sin(#phi)-sin(#phi_{mc}) (Resolution)", "tan(#lambda)-tan(#lambda_{mc}) (Resolution)", "q*(q/#it{p}_{T} - q/#it{p}_{Tmc}) (Resolution)"}; static const constexpr char* AXIS_TITLES_PULL[5] = {"#it{y}-#it{y}_{mc}/#sigma_{y} (Pull)", "#it{z}-#it{z}_{mc}/#sigma_{z} (Pull)", "sin(#phi)-sin(#phi_{mc})/#sigma_{sin(#phi)} (Pull)", "tan(#lambda)-tan(#lambda_{mc})/#sigma_{tan(#lambda)} (Pull)", "q*(q/#it{p}_{T} - q/#it{p}_{Tmc})/#sigma_{q/#it{p}_{T}} (Pull)"}; -static const constexpr char* CLUSTER_NAMES[GPUQA::N_CLS_HIST] = {"Correctly attached clusters", "Fake attached clusters", "Attached + adjacent clusters", "Fake adjacent clusters", "Clusters of reconstructed tracks", "Used in Physics", "Protected", "All clusters"}; +static const constexpr char* CLUSTER_NAMES[GPUQA::N_CLS_HIST] = {"Correctly attached clusters", "Fake attached clusters", "Attached + adjacent clusters", "Fake adjacent clusters", "Adjacent clusters (not rejected)", "Fake adjacent clusters (not rejected)", "All MC clusters of reconstructed tracks", "Clusters used in Physics", "Protected clusters", "All clusters"}; static const constexpr char* CLUSTER_TITLES[GPUQA::N_CLS_TYPE] = {"Clusters Pt Distribution / Attachment", "Clusters Pt Distribution / Attachment (relative to all clusters)", "Clusters Pt Distribution / Attachment (integrated)"}; -static const constexpr char* CLUSTER_NAMES_SHORT[GPUQA::N_CLS_HIST] = {"Attached", "Fake", "AttachAdjacent", "FakeAdjacent", "FoundTracks", "Physics", "Protected", "All"}; +static const constexpr char* CLUSTER_NAMES_SHORT[GPUQA::N_CLS_HIST] = {"Attached", "Fake", "AttachAdjacent", "FakeAdjacent", "AdjacentNoRej", "FakeAdjacentNoRej", "FoundTracks", "Physics", "Protected", "All"}; static const constexpr char* CLUSTER_TYPES[GPUQA::N_CLS_TYPE] = {"", "Ratio", "Integral"}; static const constexpr char* REJECTED_NAMES[3] = {"All", "Rejected", "Fraction"}; static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0000C0, 0x9400D3, 0x19BBBF, 0xF25900, 0x7F7F7F, 0xFFD700, 0x07F707, 0x07F7F7, 0xF08080, 0x000000}; @@ -1021,19 +1021,26 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } if (!mTrackMCLabels[i].isValid()) { for (uint32_t k = 0; k < track->NClusters(); k++) { + int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; if (mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { + if (mTracking->mIOPtrs.mergedTrackHitAttachment && gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[hitId]) == i) { + mClusterParam[hitId].rejected = 1; + } continue; } - mClusterParam[mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num].fakeAttached++; + mClusterParam[hitId].fakeAttached++; } continue; } if (mMCTrackMin == -1 || (label.getTrackID() >= mMCTrackMin && label.getTrackID() < mMCTrackMax)) { for (uint32_t k = 0; k < track->NClusters(); k++) { + int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; if (mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { + if (mTracking->mIOPtrs.mergedTrackHitAttachment && gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[hitId]) == i) { + mClusterParam[hitId].rejected = 1; + } continue; } - int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; bool correct = false; for (int32_t j = 0; j < GetMCLabelNID(hitId); j++) { if (label == GetMCLabel(hitId, j)) { @@ -1087,7 +1094,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (mClusterParam[i].attached == 0 && mClusterParam[i].fakeAttached == 0) { int32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[i]; if (attach & gputpcgmmergertypes::attachFlagMask) { - int32_t track = attach & gputpcgmmergertypes::attachTrackMask; + if ((attach & gputpcgmmergertypes::attachProtect) && !(attach & gputpcgmmergertypes::attachGoodLeg)) { + mClusterParam[i].rejected = 1; + } + int32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); mcLabelI_t trackL = mTrackMCLabels[track]; bool fake = true; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { @@ -1544,7 +1554,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (!track.OK()) { continue; } - if (!mTrackMCLabels[iTrk].isValid()) { + if (!mTrackMCLabels[iTrk].isValid()) { // Track is pure noise / bogus MC label for (uint32_t k = 0; k < track.NClusters(); k++) { if (mTracking->mIOPtrs.mergedTrackHits[track.FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { continue; @@ -1601,7 +1611,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (mMCTrackMin != -1 && (label.getTrackID() < mMCTrackMin || label.getTrackID() >= mMCTrackMax)) { continue; } - for (uint32_t k = 0; k < track.NClusters(); k++) { + for (uint32_t k = 0; k < track.NClusters(); k++) { // Clusters of reconstructed tracks if (mTracking->mIOPtrs.mergedTrackHits[track.FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { continue; } @@ -1621,8 +1631,20 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (correct) { mClusters[CL_attached]->Fill(pt, weight); mClusters[CL_tracks]->Fill(pt, weight); - } else { + } else { // cluster belongs to other particle mClusters[CL_fake]->Fill(pt, weight); + float totalWeight = 0.f; + for (int32_t j = 0; j < GetMCLabelNID(hitId); j++) { + if (!GetMCLabel(hitId, j).isFake() && GetMCTrackObj(mMCParam, GetMCLabel(hitId, j)).pt > 1.f / mTracking->GetParam().rec.maxTrackQPtB5) { + totalWeight += GetMCLabelWeight(hitId, j); + } + } + for (int32_t j = 0; j < GetMCLabelNID(hitId) && totalWeight > 0.f; j++) { + const mcLabelI_t labelCl = GetMCLabel(hitId, j); + if (!labelCl.isFake() && GetMCTrackObj(mMCParam, labelCl).pt > 1.f / mTracking->GetParam().rec.maxTrackQPtB5 && GetMCTrackObj(mRecTracks, labelCl)) { + mClusters[CL_tracks]->Fill(CAMath::Max(GetMCTrackObj(mMCParam, labelCl).pt, PT_MIN_CLUST), GetMCLabelWeight(hitId, j) * weight / totalWeight); + } + } } mClusters[CL_att_adj]->Fill(pt, weight); mClusters[CL_all]->Fill(pt, weight); @@ -1636,7 +1658,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } } - for (uint32_t i = 0; i < GetNMCLabels(); i++) { + for (uint32_t i = 0; i < GetNMCLabels(); i++) { // Loop over all clusters if ((mMCTrackMin != -1 && GetMCLabelID(i, 0) < mMCTrackMin) || (mMCTrackMax != -1 && GetMCLabelID(i, 0) >= mMCTrackMax)) { continue; } @@ -1645,9 +1667,9 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } int32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[i]; const auto& r = checkClusterState(attach); - if (mClusterParam[i].adjacent) { - int32_t label = mTracking->mIOPtrs.mergedTrackHitAttachment[i] & gputpcgmmergertypes::attachTrackMask; - if (!mTrackMCLabels[label].isValid()) { + if (mClusterParam[i].adjacent) { // Cluster is adjacent + int32_t label = gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[i]); + if (!mTrackMCLabels[label].isValid()) { // Track is pure noise / has bogus MC label float totalWeight = 0.; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { mcLabelI_t labelT = GetMCLabel(i, j); @@ -1669,6 +1691,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } mClusters[CL_att_adj]->Fill(pt, GetMCLabelWeight(i, j) * weight); mClusters[CL_fakeAdj]->Fill(pt, GetMCLabelWeight(i, j) * weight); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, GetMCLabelWeight(i, j) * weight); + mClusters[CL_fakeAdjNoReject]->Fill(pt, GetMCLabelWeight(i, j) * weight); + } mClusters[CL_all]->Fill(pt, GetMCLabelWeight(i, j) * weight); if (r.protect || r.physics) { mClusters[CL_prot]->Fill(pt, GetMCLabelWeight(i, j) * weight); @@ -1681,6 +1707,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } else { mClusters[CL_att_adj]->Fill(0.f, 1.f); mClusters[CL_fakeAdj]->Fill(0.f, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(0.f, 1.f); + mClusters[CL_fakeAdjNoReject]->Fill(0.f, 1.f); + } mClusters[CL_all]->Fill(0.f, 1.f); mClusterCounts.nUnaccessible++; if (r.protect || r.physics) { @@ -1697,6 +1727,9 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } mClusters[CL_att_adj]->Fill(pt, 1.f); mClusters[CL_tracks]->Fill(pt, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, 1.f); + } mClusters[CL_all]->Fill(pt, 1.f); if (r.protect || r.physics) { mClusters[CL_prot]->Fill(pt, 1.f); @@ -1705,7 +1738,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx mClusters[CL_physics]->Fill(pt, 1.f); } } - } else { + } else { // Cluster is not adjacent float totalWeight = 0.; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { mcLabelI_t labelT = GetMCLabel(i, j); @@ -1724,6 +1757,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx float weight = GetMCLabelWeight(i, j) / totalWeight; if (mClusterParam[i].fakeAdjacent) { mClusters[CL_fakeAdj]->Fill(pt, weight); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, weight); + mClusters[CL_fakeAdjNoReject]->Fill(pt, weight); + } } if (mClusterParam[i].fakeAdjacent) { mClusters[CL_att_adj]->Fill(pt, weight); @@ -1743,6 +1780,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } else { if (mClusterParam[i].fakeAdjacent) { mClusters[CL_fakeAdj]->Fill(0.f, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(0.f, 1.f); + mClusters[CL_fakeAdjNoReject]->Fill(0.f, 1.f); + } } if (mClusterParam[i].fakeAdjacent) { mClusters[CL_att_adj]->Fill(0.f, 1.f); @@ -1791,7 +1832,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx uint32_t jNext = 0; for (uint32_t j = 0; j < track.NClusters(); j = jNext) { uint32_t rowClCount = !(trackClusters[track.FirstClusterRef() + j].state & GPUTPCGMMergedTrackHit::flagReject); - for (jNext = j + 1; j < track.NClusters(); jNext++) { + for (jNext = j + 1; jNext < track.NClusters(); jNext++) { if (trackClusters[track.FirstClusterRef() + j].sector != trackClusters[track.FirstClusterRef() + jNext].sector || trackClusters[track.FirstClusterRef() + j].row != trackClusters[track.FirstClusterRef() + jNext].row) { break; } @@ -1898,7 +1939,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (totalWeight > 0 && 10.f * weight400 >= totalWeight) { if (!r.unattached && !r.protect && !r.physics) { mClusterCounts.nFakeRemove400++; - int32_t totalFake = weight400 < 0.9f * totalWeight; + int32_t totalFake = weight400 >= 0.9f * totalWeight; if (totalFake) { mClusterCounts.nFullFakeRemove400++; } @@ -1971,7 +2012,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx uint32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[clid]; float x = 0, y = 0, z = 0; if (attach & gputpcgmmergertypes::attachFlagMask) { - uint32_t track = attach & gputpcgmmergertypes::attachTrackMask; + uint32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); const auto& trk = mTracking->mIOPtrs.mergedTracks[track]; mTracking->GetTPCTransform()->Transform(i, j, cl.getPad(), cl.getTime(), x, y, z, trk.GetParam().GetTOffset()); mTracking->GetParam().Sector2Global(i, x, y, z, &x, &y, &z); @@ -1984,7 +2025,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx memset((void*)&trk, 0, sizeof(trk)); memset((void*)&trkHit, 0, sizeof(trkHit)); if (attach & gputpcgmmergertypes::attachFlagMask) { - uint32_t track = attach & gputpcgmmergertypes::attachTrackMask; + uint32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); trk = mTracking->mIOPtrs.mergedTracks[track]; for (uint32_t l = 0; l < trk.NClusters(); l++) { const auto& tmp = mTracking->mIOPtrs.mergedTrackHits[trk.FirstClusterRef() + l]; @@ -3310,8 +3351,8 @@ int32_t GPUQA::DoClusterCounts(uint64_t* attachClusterCounts, int32_t mode) PrintClusterCount(mode, num, "Fake Protect (< 40 MeV)", mClusterCounts.nFakeProtect40, mClusterCounts.nBelow40); } if (mcPresent() && (mQATasks & taskTrackStatistics)) { - PrintClusterCount(mode, num, "Correctly Attached all-trk normalized", mClusterCounts.nCorrectlyAttachedNormalized, mClusterCounts.nTotal); - PrintClusterCount(mode, num, "Correctly Attached non-fake normalized", mClusterCounts.nCorrectlyAttachedNormalizedNonFake, mClusterCounts.nTotal); + PrintClusterCount(mode, num, "Correctly Attached all-trk row-normalized", mClusterCounts.nCorrectlyAttachedNormalized, mClusterCounts.nTotal); + PrintClusterCount(mode, num, "Correctly Attached non-fake row-normalized", mClusterCounts.nCorrectlyAttachedNormalizedNonFake, mClusterCounts.nTotal); } if (mTextDump) { fclose(mTextDump); diff --git a/GPU/GPUTracking/qa/GPUQA.h b/GPU/GPUTracking/qa/GPUQA.h index f257c9a418659..f236498a858c6 100644 --- a/GPU/GPUTracking/qa/GPUQA.h +++ b/GPU/GPUTracking/qa/GPUQA.h @@ -143,7 +143,7 @@ class GPUQA int32_t loadHistograms(std::vector& i1, std::vector& i2, std::vector& i3, std::vector& i4, int32_t tasks = -1); void* AllocateScratchBuffer(size_t nBytes); - static constexpr int32_t N_CLS_HIST = 8; + static constexpr int32_t N_CLS_HIST = 10; static constexpr int32_t N_CLS_TYPE = 3; static constexpr int32_t MC_LABEL_INVALID = -1e9; @@ -156,7 +156,7 @@ class GPUQA }; struct additionalClusterParameters { - int32_t attached, fakeAttached, adjacent, fakeAdjacent; + int32_t attached, fakeAttached, adjacent, fakeAdjacent, rejected; float pt; }; @@ -254,7 +254,7 @@ class GPUQA std::vector mClusterParam; int32_t mNTotalFakes = 0; - TH1F* mEff[6][2][2][5]; // eff,clone,fake,all,all-fake - findable - secondaries - y,z,phi,eta,pt - work,result + TH1F* mEff[6][2][2][5]; // eff,clone,fake,all,all-fake,all-filled-internal - findable - secondaries - y,z,phi,eta,pt - work,result TGraphAsymmErrors* mEffResult[4][2][2][5]; TCanvas* mCEff[6]; TPad* mPEff[6][4]; @@ -276,11 +276,13 @@ class GPUQA CL_fake = 1, CL_att_adj = 2, CL_fakeAdj = 3, - CL_tracks = 4, - CL_physics = 5, - CL_prot = 6, - CL_all = 7 }; - TH1D* mClusters[N_CLS_TYPE * N_CLS_HIST - 1]; // attached, fakeAttached, attach+adjacent, fakeAdjacent, physics, protected, tracks, all / count, rel, integral + CL_adjNoRej = 4, + CL_fakeAdjNoReject = 5, + CL_tracks = 6, + CL_physics = 7, + CL_prot = 8, + CL_all = 9 }; + TH1D* mClusters[N_CLS_TYPE * N_CLS_HIST - 1]; // attached, fakeAttached, attach+adjacent, fakeAdjacent, adjacent not rejected, fakeAdjacent not rejected, tracks, physics, protected, all / count, rel, integral TCanvas* mCClust[N_CLS_TYPE]; TPad* mPClust[N_CLS_TYPE]; TLegend* mLClust[N_CLS_TYPE]; From 11af33091faa2345b07538c1587fba04a63823e1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 18:20:17 +0100 Subject: [PATCH 46/60] GPU TPC: Add tpcSortClusters option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx | 4 ++-- GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 4fca6292cd698..cdac865848847 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -432,6 +432,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") +AddOption(tpcSortClusters, bool, false, "", 0, "Enforce sorting TPC clusters") AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index 8ce90c740804a..485cde456f227 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -941,9 +941,9 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) std::unique_ptr tmpNativeClusterBuffer; const bool buildNativeGPU = doGPU && NeedTPCClustersOnGPU(); - const bool buildNativeHost = (mRec->GetRecoStepsOutputs() & gpudatatypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output + const bool buildNativeHost = (mRec->GetRecoStepsOutputs() & gpudatatypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().tpcSortClusters; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output const bool propagateMCLabels = buildNativeHost && GetProcessingSettings().runMC && processors()->ioPtrs.tpcPackedDigits && processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC; - const bool sortClusters = buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4); + const bool sortClusters = buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters); if (GetProcessingSettings().runMC && (!processors()->ioPtrs.tpcPackedDigits || !processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC)) { GPUWarning("Requested to process MC labels, but no labels present"); diff --git a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx index 6ca50f24351c4..fafefa7803188 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx @@ -431,7 +431,7 @@ int32_t GPUChainTracking::RunTPCDecompression() GPUMemCpy(RecoStep::TPCDecompression, mInputsHost->mPclusterNativeOutput, DecompressorShadow.mNativeClustersBuffer, sizeof(Decompressor.mNativeClustersBuffer[0]) * nClustersFinal, unattachedStream, false); SynchronizeStream(unattachedStream); } - if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4) { + if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters) { runKernel(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression)); const ClusterNativeAccess* decoded = mIOPtrs.clustersNative; if (doGPU) { From 58cef7f345402620b79c2a10ff9c954e7c08c0c4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 19:02:39 +0100 Subject: [PATCH 47/60] GPU Standalone / QA: Add testSyncAsyncMC option to pass cluster MC labels to async pass --- .../DataCompression/GPUTPCCompression.h | 1 + .../GPUTPCCompressionKernels.cxx | 8 ++- GPU/GPUTracking/DataTypes/GPUDataTypesIO.h | 1 + .../Definitions/GPUSettingsListStandalone.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.h | 1 + .../Global/GPUChainTrackingCompression.cxx | 68 ++++++++++++++----- .../Standalone/Benchmark/standalone.cxx | 21 ++++-- 7 files changed, 76 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompression.h b/GPU/GPUTracking/DataCompression/GPUTPCCompression.h index 852156d7ab7f0..b628af3c7dde1 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompression.h +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompression.h @@ -65,6 +65,7 @@ class GPUTPCCompression : public GPUProcessor void DumpCompressedClusters(std::ostream& out); #endif GPUd() bool rejectCluster(int32_t idx, const GPUParam& param, const GPUTrackingInOutPointers& ioPtrs) const; + GPUd() static bool isClusterRejected(int32_t idx, const GPUParam& param, const GPUTrackingInOutPointers& ioPtrs); protected: struct memory { diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 67dacf993f0a4..588340bd73e64 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -189,9 +189,11 @@ GPUd() bool GPUTPCCompressionKernels::GPUTPCCompressionKernels_Compare<4>::opera GPUd() bool GPUTPCCompression::rejectCluster(int32_t idx, const GPUParam& GPUrestrict() param, const GPUTrackingInOutPointers& GPUrestrict() ioPtrs) const { - if (mClusterStatus[idx]) { - return true; - } + return mClusterStatus[idx] || isClusterRejected(idx, param, ioPtrs); +} + +GPUd() bool GPUTPCCompression::isClusterRejected(int32_t idx, const GPUParam& GPUrestrict() param, const GPUTrackingInOutPointers& GPUrestrict() ioPtrs) +{ int32_t attach = ioPtrs.mergedTrackHitAttachment[idx]; bool unattached = attach == 0; diff --git a/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h b/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h index b541df40a23e5..dd93996078163 100644 --- a/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h +++ b/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h @@ -195,6 +195,7 @@ struct GPUTrackingInOutPointers { uint32_t nOutputClusRefsTPCO2 = 0; const o2::MCCompLabel* outputTracksTPCO2MC = nullptr; const o2::tpc::CompressedClustersFlat* tpcCompressedClusters = nullptr; + const o2::dataformats::ConstMCTruthContainerView* tpcCompressedClustersMC = nullptr; const o2::tpc::ClusterNativeAccess* clustersNativeReduced = nullptr; // TPC links diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h index 6ff3f8ec38487..be67272379b10 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h @@ -82,6 +82,7 @@ AddOption(printSettings, int32_t, 0, "", 0, "Print all settings", def(1)) AddOption(testSyncAsync, bool, false, "syncAsync", 0, "Test first synchronous and then asynchronous processing") AddOption(testSync, bool, false, "sync", 0, "Test settings for synchronous phase") AddOption(testSyncAsyncQcInSync, bool, false, "syncAsyncSyncQC", 0, "Run QC in sync phase of testSyncAsync") +AddOption(testSyncAsyncMC, bool, false, "", 0, "Propagate the TPC cluster MC labels of the clusters surviving the compression to the asynchronous phase of testSyncAsync (enforces tpcSortClusters)") AddOption(timeFrameTime, bool, false, "tfTime", 0, "Print some debug information about time frame processing time") AddOption(controlProfiler, bool, false, "", 0, "Issues GPU profiler stop and start commands to profile only the relevant processing part") AddOption(preloadEvents, bool, false, "", 0, "Preload events into host memory before start processing") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 92eda50194c74..1fef646f803c4 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -159,6 +159,7 @@ class GPUChainTracking : public GPUChain const GPUQA* GetQA() const { return mQAFromForeignChain ? mQAFromForeignChain->mQA.get() : mQA.get(); } GPUQA* GetQA() { return mQAFromForeignChain ? mQAFromForeignChain->mQA.get() : mQA.get(); } int32_t ForceInitQA(); + void GenerateCompressionMCLabels(const GPUChainTracking& chainSync); void SetQAFromForeignChain(GPUChainTracking* chain) { mQAFromForeignChain = chain; } const GPUSettingsDisplay* GetEventDisplayConfig() const { return mConfigDisplay; } const GPUSettingsQA* GetQAConfig() const { return mConfigQA; } diff --git a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx index fafefa7803188..5d7a785b7a25e 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx @@ -253,7 +253,7 @@ int32_t GPUChainTracking::RunTPCDecompression() auto& decompressTimer = getTimer("TPCDecompression", 0); auto allocatorUse = runFiltering ? std::function{allocatorTmp} : std::function{allocatorFinal}; decompressTimer.Start(); - if (decomp.decompress(mIOPtrs.tpcCompressedClusters, *mClusterNativeAccess, allocatorUse, param(), GetProcessingSettings().deterministicGPUReconstruction)) { + if (decomp.decompress(mIOPtrs.tpcCompressedClusters, *mClusterNativeAccess, allocatorUse, param(), GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters)) { GPUError("Error decompressing clusters"); return 1; } @@ -433,15 +433,8 @@ int32_t GPUChainTracking::RunTPCDecompression() } if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters) { runKernel(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression)); - const ClusterNativeAccess* decoded = mIOPtrs.clustersNative; if (doGPU) { - for (uint32_t i = 0; i < NSECTORS; i++) { - for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - ClusterNative* begin = mInputsHost->mPclusterNativeOutput + decoded->clusterOffset[i][j]; - ClusterNative* end = begin + decoded->nClusters[i][j]; - std::sort(begin, end); - } - } + SortClusters(false, false, mClusterNativeAccess.get(), mInputsHost->mPclusterNativeOutput); } SynchronizeStream(unattachedStream); } @@ -453,10 +446,58 @@ int32_t GPUChainTracking::RunTPCDecompression() tmpParam.reset(); WriteConstantParams(); } + if (mIOPtrs.tpcCompressedClustersMC) { + if (mIOPtrs.tpcCompressedClustersMC->getIndexedSize() != mIOPtrs.clustersNative->nClustersTotal) { + GPUError("Number of MC labels for compressed clusters (%u) does not match number of decoded clusters (%u)", (uint32_t)mIOPtrs.tpcCompressedClustersMC->getIndexedSize(), (uint32_t)mIOPtrs.clustersNative->nClustersTotal); + return 1; + } + mClusterNativeAccess->clustersMCTruth = mIOPtrs.tpcCompressedClustersMC; + } DoDebugDump(GPUChainTrackingDebugFlags::TPCDecompressedClusters, &GPUChainTracking::DumpClusters, *mDebugFile, mIOPtrs.clustersNative); return 0; } +namespace +{ +template +void writeReducedClusterLabels(const ClusterNativeAccess& clusters, std::pair labelBuffer, T&& rejectCluster) +{ + o2::dataformats::MCLabelContainer tmpContainer; + uint32_t nOutput = 0; + for (uint32_t i = 0; i < clusters.nClustersTotal; i++) { + if (!rejectCluster(i)) { + for (const auto& element : clusters.clustersMCTruth->getLabels(i)) { + tmpContainer.addElement(nOutput, element); + } + nOutput++; + } + } + tmpContainer.flatten_to(*labelBuffer.first); + *labelBuffer.second = *labelBuffer.first; +} +} // namespace + +void GPUChainTracking::GenerateCompressionMCLabels(const GPUChainTracking& chainSync) +{ + const GPUTrackingInOutPointers& ioPtrs = chainSync.mIOPtrs; + if (!ioPtrs.clustersNative || !ioPtrs.clustersNative->clustersMCTruth || !ioPtrs.mergedTrackHitAttachment || !ioPtrs.mergedTracks) { + throw std::runtime_error("Cannot generate MC labels for compressed clusters, need clusters with MC labels and merged tracks with cluster attachment of the sync chain"); + } + for (uint32_t i = 0; i < NSECTORS; i++) { // The decoded clusters get sorted, so the labels match only if the clusters before the compression were sorted as well + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + for (uint32_t k = 1; k < ioPtrs.clustersNative->nClusters[i][j]; k++) { + if (ioPtrs.clustersNative->clusters[i][j][k] < ioPtrs.clustersNative->clusters[i][j][k - 1]) { + throw std::runtime_error("Cannot generate MC labels for compressed clusters, clusters of sync chain are not sorted"); + } + } + } + } + mIOMem.clusterNativeMCView = std::make_unique(); + mIOMem.clusterNativeMCBuffer = std::make_unique(); + writeReducedClusterLabels(*ioPtrs.clustersNative, {mIOMem.clusterNativeMCBuffer.get(), mIOMem.clusterNativeMCView.get()}, [&chainSync, &ioPtrs](uint32_t i) { return GPUTPCCompression::isClusterRejected(i, chainSync.GetParam(), ioPtrs); }); + mIOPtrs.tpcCompressedClustersMC = mIOMem.clusterNativeMCView.get(); +} + void GPUChainTracking::WriteReducedClusters() { GPUTPCCompression& Compressor = processors()->tpcCompressor; @@ -491,21 +532,14 @@ void GPUChainTracking::WriteReducedClusters() } nOutput = 0; - o2::dataformats::MCLabelContainer tmpContainer; for (uint32_t i = 0; i < mIOPtrs.clustersNative->nClustersTotal; i++) { if (!Compressor.rejectCluster(i, param(), mIOPtrs)) { - if (mIOPtrs.clustersNative->clustersMCTruth) { - for (const auto& element : mIOPtrs.clustersNative->clustersMCTruth->getLabels(i)) { - tmpContainer.addElement(nOutput, element); - } - } clBuffer[nOutput++] = mIOPtrs.clustersNative->clustersLinear[i]; } } mIOPtrs.clustersNativeReduced = mClusterNativeAccessReduced.get(); if (mIOPtrs.clustersNative->clustersMCTruth) { - tmpContainer.flatten_to(*labelBuffer.first); - *labelBuffer.second = *labelBuffer.first; + writeReducedClusterLabels(*mIOPtrs.clustersNative, labelBuffer, [this, &Compressor](uint32_t i) { return Compressor.rejectCluster(i, param(), mIOPtrs); }); mClusterNativeAccessReduced->clustersMCTruth = labelBuffer.second; } } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 652ec5ccc23f2..925b2c584857e 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -208,6 +208,13 @@ int32_t ReadConfiguration(int argc, char** argv) printf("Cannot run asynchronous processing with double pipeline\n"); return 1; } + if (configStandalone.testSyncAsyncMC) { + if (!configStandalone.testSyncAsync || configStandalone.QA.noMC) { + printf("testSyncAsyncMC requires syncAsync and cannot be used with QA.noMC\n"); + return 1; + } + configStandalone.proc.tpcSortClusters = true; + } if (configStandalone.proc.doublePipeline && (configStandalone.runs < 4 || !configStandalone.outputcontrolmem)) { printf("Double pipeline mode needs at least 4 runs per event and external output. To cycle though multiple events, use --preloadEvents and --runs n for n iterations round-robin\n"); return 1; @@ -531,7 +538,7 @@ int32_t SetupReconstruction() steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); steps.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); - procSet.runMC = false; + procSet.runMC = configStandalone.testSyncAsyncMC; procSet.runQA = runAsyncQA; procSet.eventDisplay = eventDisplay.get(); procSet.runCompressionStatistics = 0; @@ -598,10 +605,11 @@ int32_t ReadEvent(int32_t n) } #if !defined(GPUCA_RUN2) && defined(GPUCA_BUILD_QA) && defined(GPUCA_STANDALONE) if ((configStandalone.proc.runQA || configStandalone.eventDisplay) && !configStandalone.QA.noMC) { - chainTracking->ForceInitQA(); - chainTracking->GetQA()->UpdateChain(chainTracking); - if (chainTracking->GetQA()->ReadO2MCData((eventsDir + "mc." + std::to_string(n) + ".dump").c_str()) && - chainTracking->GetQA()->ReadO2MCData((eventsDir + "mc.0.dump").c_str()) && configStandalone.proc.runQA) { + GPUChainTracking* chain = (configStandalone.testSyncAsyncMC && !configStandalone.testSyncAsyncQcInSync) ? chainTrackingAsync : chainTracking; + chain->ForceInitQA(); + chain->GetQA()->UpdateChain(chain); + if (chain->GetQA()->ReadO2MCData((eventsDir + "mc." + std::to_string(n) + ".dump").c_str()) && + chain->GetQA()->ReadO2MCData((eventsDir + "mc.0.dump").c_str()) && configStandalone.proc.runQA) { throw std::runtime_error("Error reading O2 MC dump"); } } @@ -753,6 +761,9 @@ int32_t RunBenchmark(GPUReconstruction* recUse, GPUChainTracking* chainTrackingU chainTrackingAsync->mIOPtrs.nRawClusters[i] = 0; } chainTrackingAsync->mIOPtrs.clustersNative = nullptr; + if (configStandalone.testSyncAsyncMC) { + chainTrackingAsync->GenerateCompressionMCLabels(*chainTracking); + } if (configStandalone.runsInit > 0 && configStandalone.proc.debugCSV.empty()) { recAsync->SetResetTimers(iRun < configStandalone.runsInit); } From 4dd813d188555d983a75cf2209c74d73eab58ac6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 22:28:25 +0100 Subject: [PATCH 48/60] GPU Standalone: add testSyncAsyncNoSyncSettings option --- .../Definitions/GPUSettingsListStandalone.h | 1 + .../Standalone/Benchmark/standalone.cxx | 80 ++++++++++--------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h index be67272379b10..20018a2792cc0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h @@ -82,6 +82,7 @@ AddOption(printSettings, int32_t, 0, "", 0, "Print all settings", def(1)) AddOption(testSyncAsync, bool, false, "syncAsync", 0, "Test first synchronous and then asynchronous processing") AddOption(testSync, bool, false, "sync", 0, "Test settings for synchronous phase") AddOption(testSyncAsyncQcInSync, bool, false, "syncAsyncSyncQC", 0, "Run QC in sync phase of testSyncAsync") +AddOption(testSyncAsyncNoSyncSettings, bool, false, "", 0, "Do not apply the settings for synchronous processing in testSyncAsync / testSync") AddOption(testSyncAsyncMC, bool, false, "", 0, "Propagate the TPC cluster MC labels of the clusters surviving the compression to the asynchronous phase of testSyncAsync (enforces tpcSortClusters)") AddOption(timeFrameTime, bool, false, "tfTime", 0, "Print some debug information about time frame processing time") AddOption(controlProfiler, bool, false, "", 0, "Issues GPU profiler stop and start commands to profile only the relevant processing part") diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 925b2c584857e..62d53f7bb9580 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -453,6 +453,7 @@ int32_t SetupReconstruction() } steps.steps = gpudatatypes::RecoStep::AllRecoSteps; + steps.outputs.clear(); if (configStandalone.runTRD != -1) { steps.steps.setBits(gpudatatypes::RecoStep::TRDTracking, configStandalone.runTRD > 0); } else if (chainTracking->GetTRDGeometry() == nullptr) { @@ -485,7 +486,6 @@ int32_t SetupReconstruction() steps.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); steps.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); } else if (grp.needsClusterer) { steps.inputs.setBits(gpudatatypes::InOutType::TPCRaw, true); steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); @@ -493,8 +493,12 @@ int32_t SetupReconstruction() steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); } + GPUSettingsRec recSetAsync = recSet; + GPUSettingsProcessing procSetAsync = procSet; + GPURecoStepConfiguration stepsAsync = steps; + // Set settings for synchronous - GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, configStandalone.testSyncAsync || configStandalone.testSync, configStandalone.rundEdx); + GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, (configStandalone.testSyncAsync || configStandalone.testSync) && !configStandalone.testSyncAsyncNoSyncSettings, configStandalone.rundEdx); int32_t runAsyncQA = procSet.runQA && !configStandalone.testSyncAsyncQcInSync ? procSet.runQA : 0; if (configStandalone.testSyncAsync) { procSet.eventDisplay = nullptr; @@ -503,26 +507,26 @@ int32_t SetupReconstruction() } } - // Apply --recoSteps flag last so it takes precedence - // E.g. ApplySyncSettings might enable TPCdEdx, but might not be needed if only clusterizer was requested - if (configStandalone.recoSteps >= 0) { - steps.steps &= configStandalone.recoSteps; - } - if (configStandalone.recoStepsGPU >= 0) { - steps.stepsGPUMask &= configStandalone.recoStepsGPU; - } + const auto finalizeSteps = [](GPURecoStepConfiguration& stepsCfg, GPUSettingsProcessing& procCfg) { + if (configStandalone.recoSteps >= 0) { + stepsCfg.steps &= configStandalone.recoSteps; + } + if (configStandalone.recoStepsGPU >= 0) { + stepsCfg.stepsGPUMask &= configStandalone.recoStepsGPU; + } - steps.outputs.clear(); - steps.outputs.setBits(gpudatatypes::InOutType::TPCMergedTracks, steps.steps.isSet(gpudatatypes::RecoStep::TPCMerging)); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, steps.steps.isSet(gpudatatypes::RecoStep::TPCCompression)); - steps.outputs.setBits(gpudatatypes::InOutType::TRDTracks, steps.steps.isSet(gpudatatypes::RecoStep::TRDTracking)); - steps.outputs.setBits(gpudatatypes::InOutType::TPCClusters, steps.steps.isSet(gpudatatypes::RecoStep::TPCClusterFinding)); - - if (steps.steps.isSet(gpudatatypes::RecoStep::TRDTracking)) { - if (procSet.createO2Output && !procSet.trdTrackModelO2) { - procSet.createO2Output = 1; // Must not be 2, to make sure TPC GPU tracks are still available for TRD + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCMergedTracks, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCMerging)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCCompression)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TRDTracks, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TRDTracking)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCClusters, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCClusterFinding) || stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCDecompression)); + + if (stepsCfg.steps.isSet(gpudatatypes::RecoStep::TRDTracking)) { + if (procCfg.createO2Output && !procCfg.trdTrackModelO2) { + procCfg.createO2Output = 1; // Must not be 2, to make sure TPC GPU tracks are still available for TRD + } } - } + }; + finalizeSteps(steps, procSet); rec->SetSettings(&grp, &recSet, &procSet, &steps); if (configStandalone.proc.doublePipeline) { @@ -530,25 +534,25 @@ int32_t SetupReconstruction() } if (configStandalone.testSyncAsync) { // TODO: Add --async mode / flag // Set settings for asynchronous - steps.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); - steps.steps.setBits(gpudatatypes::RecoStep::TPCdEdx, true); - steps.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); - steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCRaw, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); - procSet.runMC = configStandalone.testSyncAsyncMC; - procSet.runQA = runAsyncQA; - procSet.eventDisplay = eventDisplay.get(); - procSet.runCompressionStatistics = 0; - if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { - procSet.tpcInputWithClusterRejection = 1; + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCdEdx, true); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCRaw, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); + procSetAsync.runMC = configStandalone.testSyncAsyncMC; + procSetAsync.runQA = runAsyncQA; + procSetAsync.eventDisplay = eventDisplay.get(); + procSetAsync.runCompressionStatistics = 0; + if (recSetAsync.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { + procSetAsync.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableMarkAdjacent = 0xFF; - recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); - GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); - recAsync->SetSettings(&grp, &recSet, &procSet, &steps); + recSetAsync.tpc.disableMarkAdjacent = 0xFF; + recSetAsync.maxTrackQPtB5 = CAMath::Min(recSetAsync.maxTrackQPtB5, recSetAsync.tpc.rejectQPtB5); + GPUChainTracking::ApplySyncSettings(procSetAsync, recSetAsync, stepsAsync.steps, false, configStandalone.rundEdx); + finalizeSteps(stepsAsync, procSetAsync); + recAsync->SetSettings(&grp, &recSetAsync, &procSetAsync, &stepsAsync); } if (configStandalone.outputcontrolmem) { From ab9616183684122a4f65d1d542f518181bbb375b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 23:06:41 +0100 Subject: [PATCH 49/60] GPU QA: Add extra histograms for pt > 50 / 200 MeV --- GPU/GPUTracking/qa/GPUQA.cxx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 13bafef79312d..bd0deed3f0fd3 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2825,7 +2825,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - uint64_t attachClusterCounts[N_CLS_HIST]; + uint64_t attachClusterCounts[N_CLS_HIST + 4]; // + correctly attached / all clusters in the pt bins entirely above 50 MeV and above 200 MeV if (mQATasks & taskClusterAttach) { // Process Cluster Attachment Histograms if (mConfig.inputHistogramsOnly == 0) { @@ -2849,6 +2849,17 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } attachClusterCounts[i] = val; } + for (int32_t i = 0; i < 2; i++) { + double valAttached = 0, valAll = 0; + for (int32_t j = 1; j < mClusters[CL_all]->GetXaxis()->GetNbins() + 2; j++) { + if (mClusters[CL_all]->GetXaxis()->GetBinLowEdge(j) >= (i ? 0.2 : 0.05)) { + valAttached += mClusters[CL_attached]->GetBinContent(j); + valAll += mClusters[CL_all]->GetBinContent(j); + } + } + attachClusterCounts[N_CLS_HIST + 2 * i] = valAttached; + attachClusterCounts[N_CLS_HIST + 2 * i + 1] = valAll; + } if (!CLUST_HIST_INT_SUM) { for (int32_t i = 0; i < N_CLS_HIST; i++) { @@ -3324,6 +3335,8 @@ int32_t GPUQA::DoClusterCounts(uint64_t* attachClusterCounts, int32_t mode) PrintClusterCount(mode, num, "Unattached", attachClusterCounts[N_CLS_HIST - 1] - attachClusterCounts[CL_att_adj], mClusterCounts.nTotal); PrintClusterCount(mode, num, "Removed (Strategy A)", attachClusterCounts[CL_att_adj] - attachClusterCounts[CL_prot], mClusterCounts.nTotal); // Attached + Adjacent (also fake) - protected PrintClusterCount(mode, num, "Unaccessible", mClusterCounts.nUnaccessible, mClusterCounts.nTotal); // No contribution from track >= 10 MeV, unattached or fake-attached/adjacent + PrintClusterCount(mode, num, "Correctly attached (> 50 MeV)", attachClusterCounts[N_CLS_HIST], attachClusterCounts[N_CLS_HIST + 1]); // Normalized to all clusters of particles above the threshold + PrintClusterCount(mode, num, "Correctly attached (> 200 MeV)", attachClusterCounts[N_CLS_HIST + 2], attachClusterCounts[N_CLS_HIST + 3]); } else { PrintClusterCount(mode, num, "All Clusters", mClusterCounts.nTotal, mClusterCounts.nTotal); PrintClusterCount(mode, num, "Used in Physics", mClusterCounts.nPhysics, mClusterCounts.nTotal); From 63b9f6423dea9318778692b83e6d5c60108c4100 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 14:45:25 +0100 Subject: [PATCH 50/60] GPU Math: Add support for half-precision min/max in CUDA --- GPU/Common/GPUCommonMath.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 62efdfb80bb35..2e542d027307f 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -236,6 +236,19 @@ class GPUCommonMath typedef GPUCommonMath CAMath; +#ifdef __CUDACC__ +template <> +GPUhd() half GPUCommonMath::Min(const half x, const half y) // BUG: CUDA __hmin / __hmax is not constexpr, fix once fixed in CUDA +{ + return __hmin(x, y); +} +template <> +GPUhd() half GPUCommonMath::Max(const half x, const half y) +{ + return __hmax(x, y); +} +#endif + template GPUhdi() constexpr float GPUCommonMath::Sum2(float w, Args... args) { From 8b41d58f2491f47f333ddab7c4cb39fae98ac28b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 23:33:33 +0100 Subject: [PATCH 51/60] GPU CPUDevice: Add consistency check for some RTC parameters --- .../Base/cpudevice/GPUReconstructionCPUDevice.cxx | 12 +++++++++--- .../cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx index 846dd32674be9..149bb75c52792 100644 --- a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx +++ b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx @@ -39,9 +39,7 @@ GPUReconstructionCPUDevice::GPUReconstructionCPUDevice(const GPUSettingsDeviceBa { if (mMaster == nullptr) { mInternals = new GPUReconstructionCPUDeviceInternals; - *mParDevice = *mParCPU; // We run the CPU kernels on the "device", so we use the CPU kernel parameters - mParDevice->par_ALTERNATE_BORDER_SORT = 1; // Required by the GPU code path without RTC - mParDevice->par_CF_SCAN_WORKGROUP_SIZE = 1; // No block-wide scan with 1 thread per block, the clusterizer uses the CPU stream compaction + *mParDevice = *mParCPU; // We run the host-compiled CPU kernels on the "device", so the device parameters must be the CPU kernel parameters. Other values are possible only with RTC (genRTC), checked in InitDevice_Runtime for (auto& v : mParDevice->par_LB_maxThreads) { v = 1; } @@ -72,6 +70,14 @@ int32_t GPUReconstructionCPUDevice::InitDevice_Runtime() exit(0); } + if (mProcessingSettings->alternateBorderSort == -1) { + mProcessingSettings->alternateBorderSort = 1; // Required by the GPU code path without RTC + } + if (!GetProcessingSettings().rtc.enable && memcmp(&mParDevice->par_LB_forceBlocks + 1, &mParCPU->par_LB_forceBlocks + 1, (const char*)(mParCPU + 1) - (const char*)(&mParCPU->par_LB_forceBlocks + 1))) { // All members behind the launch bounds + GPUError("CPUDevice kernel parameters differ from the parameters the host kernels were compiled with, this is possible only with RTC"); + return 1; + } + if (mMaster == nullptr) { mMultiprocessorCount = std::max(1, mMaxHostThreads); // One device block runs as one TBB task mThreadCount = 1; diff --git a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx index 90a765b85cba3..afa6b24098972 100644 --- a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx +++ b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx @@ -60,6 +60,7 @@ int32_t GPUReconstructionCPUDevice::genRTC(std::string& filename, uint32_t& nCom const std::string baseCommand = rtc::baseCommand(proc, std::string_view(_binary_GPUReconstructionCPUDevicertc_command_start, _binary_GPUReconstructionCPUDevicertc_command_len)) + " " + proc.rtctech.overrideArchitecture; const std::string noFastMath(_binary_GPUReconstructionCPUDevicertc_command_no_fast_math_start, _binary_GPUReconstructionCPUDevicertc_command_no_fast_math_len); + mParDevice->par_CF_SCAN_WORKGROUP_SIZE = 1; // Must match the 1 thread per block for the scan kernels to compile, the clusterizer uses the CPU stream compaction rtc::loadLaunchBounds(proc, *mParDevice); const std::string launchBounds = rtc::launchBoundsCode(proc, *mParDevice, 0, mWarpSize); if (proc.rtctech.printLaunchBounds >= 2) { From 536989300e765e10156aa8b70e355c2e88a77b3c Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 11:51:24 +0100 Subject: [PATCH 52/60] GPU TPC: synchronize threads in warp between fit iterations to reduce warp divergence --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 1 + 1 file changed, 1 insertion(+) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a8b13a7bfe480..7fd2414a6365e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -94,6 +94,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int8_t adjacentLeg = 1 + (track.Leg() == 0); for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? + GPUbarrierWarp(); // TODO: Validate on different GPUs, check if we should do similar things in other places int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; From 9f042a0768897f8fe35026641a1e193720c87283 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 19:17:43 +0100 Subject: [PATCH 53/60] GPU: Provide separate fit kernels for rebuild/retry, remove mergerSortTracks runtime option --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 3 --- GPU/GPUTracking/Definitions/GPUGetConstexpr.h | 2 +- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Definitions/Parameters/GPUParameters.csv | 4 +++- .../Parameters/GPUParameters.json.example | 16 ++++++++++++- .../Global/GPUChainTrackingMerger.cxx | 8 +++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +-- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 24 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 5 +++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 16 ++++++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +++-- GPU/GPUTracking/kernels.cmake | 4 +++- 12 files changed, 67 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index 43af7a2cad438..d644235eead76 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -301,9 +301,6 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice() if (!GetProcessingSettings().createO2Output || !IsGPU()) { mProcessingSettings->clearO2OutputFromGPU = false; } - if (!(mRecoSteps.stepsGPUMask & gpudatatypes::RecoStep::TPCMerging)) { - mProcessingSettings->mergerSortTracks = false; - } if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) { mProcessingSettings->delayedOutput = false; } diff --git a/GPU/GPUTracking/Definitions/GPUGetConstexpr.h b/GPU/GPUTracking/Definitions/GPUGetConstexpr.h index b70890738e1b7..5d6d9613e54bd 100644 --- a/GPU/GPUTracking/Definitions/GPUGetConstexpr.h +++ b/GPU/GPUTracking/Definitions/GPUGetConstexpr.h @@ -22,7 +22,7 @@ // This is a temporary workaround required for clang (with c++20), until we can go to C++23 with P2280R4, which allows getting constexpr static values from references -#if defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L +#if (defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L) || defined(__CPUDEVICE__) // BUG: The workaround is needed for GCC if used inside a templated function. namespace o2::gpu::internal { diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index cdac865848847..56d7b004c9212 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -397,7 +397,6 @@ AddOption(nStreams, int8_t, 8, "", 0, "Number of GPU streams / command queues") AddOption(nTPCClustererLanes, int8_t, -1, "", 0, "Number of TPC clusterers that can run in parallel (-1 = autoset)") AddOption(overrideClusterizerFragmentLen, int32_t, -1, "", 0, "Force the cluster max fragment len to a certain value (-1 = autodetect)") AddOption(delayedOutput, bool, true, "", 0, "Delay output to be parallel to track fit") -AddOption(mergerSortTracks, int8_t, -1, "", 0, "Sort track indizes for GPU track fit (-1 = auto-device based on device parameters)") AddOption(alternateBorderSort, int8_t, -1, "", 0, "Alternative implementation for sorting of border tracks (-1 = auto-device based on device parameters)") AddOption(tpcCompressionGatherMode, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode (0: DMA transfer gather gpu to host, 1: serial DMA to host and gather by copy on CPU, 2. gather via GPU kernal DMA access, 3. gather on GPU via kernel, dma afterwards, -1 = auto-device based on device parameters)") AddOption(tpcCompressionGatherModeKernel, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode Kernel (0: unbufferd, 1-3: buffered, 4: multi-block, -1 = auto-device based on device parameters)") diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 5892f4acfe9c4..b197c20130738 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -27,7 +27,9 @@ GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_ GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,"[""GPUCA_WARP_SIZE"", 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]", GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,"[928, 1]",,"[1024, 1]","[320, 2]", COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,,, -GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_def,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_rebuild,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_retry,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,"[128, 16]",,"[224, 3]","[256, 7]","[32, 64]" GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,"[32, 20]",,"[32, 10]","[64, 4]","[256, 8]" GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,256,,256,256, diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example index 15f905cbcc41e..7390c4b520ae9 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example @@ -167,7 +167,21 @@ "AMPERE": 1024, "TURING": 1024 }, - "GPUTPCGMMergerTrackFit": { + "GPUTPCGMMergerTrackFit_def": { + "default": 256, + "MI100": [192, 2], + "VEGA": [64, 7], + "AMPERE": [64, 4], + "TURING": [32, 8] + }, + "GPUTPCGMMergerTrackFit_rebuild": { + "default": 256, + "MI100": [192, 2], + "VEGA": [64, 7], + "AMPERE": [64, 4], + "TURING": [32, 8] + }, + "GPUTPCGMMergerTrackFit_retry": { "default": 256, "MI100": [192, 2], "VEGA": [64, 7], diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 6786f7fc927e9..3fbf2453fe700 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -196,7 +196,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) waitForTransfer = 1; } - const bool mergerSortTracks = GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT : GetProcessingSettings().mergerSortTracks; + const bool mergerSortTracks = mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT; if (mergerSortTracks) { runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); @@ -236,7 +236,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); @@ -250,10 +250,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); } if (param().rec.tpc.retryRefit) { - runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + runKernel(GetGridAuto(0)); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 31a6b9a18a10b..614f73679180f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -430,8 +430,7 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) { computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks); - const bool mergerSortTracks = mRec->GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT : mRec->GetProcessingSettings().mergerSortTracks; - if (mergerSortTracks) { + if (mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT) { computePointerWithAlignment(mem, mTrackOrderProcess, mNMaxTracks); } return mem; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 2ef0d0ba59748..48780cb060d90 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -18,16 +18,30 @@ using namespace o2::gpu; -template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) +template +GPUdii() void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + constexpr bool rebuilt = I == GPUTPCGMMergerTrackFit::rebuild, retry = I == GPUTPCGMMergerTrackFit::retry; + const int32_t iEnd = retry ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); + int32_t i; + if constexpr (retry) { + i = merger.RetryRefitIds()[ii]; + } else if constexpr (GPUCA_PAR_SORT_BEFORE_FIT > 0) { + i = merger.TrackOrderProcess()[ii]; + } else { + i = ii; + } + (GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger)); // Parentheses, since the comma of the template arguments would split the macro arguments }); } +#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, the kernels are called from another compilation unit +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +#endif + template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 82605f47a5f27..023a72f276de3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -35,8 +35,11 @@ class GPUTPCGMMergerGeneral : public GPUKernelTemplate class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: + enum K { def = 0, + rebuild = 1, + retry = 2 }; template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7fd2414a6365e..46254350d1393 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -56,11 +56,13 @@ static thread_local bool CADEBUG_CONDITION = true; using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) +template +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; const GPUParam& GPUrestrict() param = merger.Param(); + const bool rebuilt = retryAttempt ? param.rec.tpc.rebuildTrackInFit : rebuiltKernel; // The retry runs on the rebuilt tracks if the rebuild is enabled const float maxSinForUpdate = param.rec.tpc.maxSinPhiForFitUpdate; GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -1488,11 +1490,13 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) +template +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger) { if (!track.OK()) { return; } + const bool rebuilt = retryAttempt ? merger.Param().rec.tpc.rebuildTrackInFit : rebuiltKernel; CADEBUG3(if (GPUCA_CADEBUG_ENABLED >= 2) { static const std::set dbgTracks = []() { std::set m; if (const char* e = getenv("DEBUG_TRACK")) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } return m; }(); CADEBUG_CONDITION = dbgTracks.count(iTrk); }); @@ -1500,7 +1504,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { @@ -1536,6 +1540,12 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() // if (track.OK()) merger.DebugRefitMergedTrack(track); } +#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, RefitTrack is called from another compilation unit +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +#endif + GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) { float cA, sA; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 2e8ee86ca8196..665c2da4ef14d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,14 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); + template + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); + template + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 539840baaff7a..ade04a4bdfeb7 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,9 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, def" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, rebuild" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, retry" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From 565203e95c06f06d324443dcf76238accd6b7945 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 08:15:13 +0100 Subject: [PATCH 54/60] GPU TPC: Refactor rebuild data structures to remove GPU memory usage --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 4 +- .../Global/GPUChainTrackingMerger.cxx | 10 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 92 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 29 +++--- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 13 +-- .../Merger/GPUTPCGMMergerRebuildInternal.h | 64 +++++++++++++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 30 +++--- GPU/GPUTracking/kernels.cmake | 2 +- 8 files changed, 172 insertions(+), 72 deletions(-) create mode 100644 GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 2761fcb3ec733..b66cc7d49dcc9 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -258,8 +258,8 @@ bool GPUChainTracking::ValidateSteps() bool GPUChainTracking::ValidateSettings() { int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCCompression).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode; - if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { - GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); + if (param().rec.tpc.rebuildTrackInFit && (param().rec.tpc.rebuildTrackInFitClusterCandidates == 0 || param().rec.tpc.rebuildTrackInFitClusterCandidates > 13)) { + GPUError("Need between 1 and 13 cluster candidates to rebuild tracks during fit"); return false; } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 3fbf2453fe700..b9804a767ad2e 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -19,6 +19,7 @@ #include "GPUO2DataTypes.h" #include "GPUQA.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUConstantMem.h" #include "GPUTPCGMMergerGPU.h" #include "GPUTPCGMO2Output.h" @@ -206,6 +207,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } + if (param().rec.tpc.rebuildTrackInFit) { + mRec->PushNonPersistentMemory(qStr2Tag("TPCREBLD")); // The rebuild memory is freed after the fit, before the O2 output + } runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); @@ -233,7 +237,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (param().rec.tpc.rebuildTrackInFit) { + AllocateRegisteredMemory(Merger.MemoryResRebuild()); // Sized by the number of merged tracks, which was transferred to the host above + WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * sizeof(*MergerShadow.TrackRebuildHelperRow())); } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); @@ -255,6 +262,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (param().rec.tpc.retryRefit) { runKernel(GetGridAuto(0)); } + if (param().rec.tpc.rebuildTrackInFit) { + mRec->PopNonPersistentMemory(RecoStep::TPCMerging, qStr2Tag("TPCREBLD")); + } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 614f73679180f..26d83fdfa77a3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -34,6 +34,7 @@ #include "GPUTPCTracker.h" #include "GPUTPCTrackParam.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUO2DataTypes.h" #include "TPCFastTransformPOD.h" #include "GPUTPCConvertImpl.h" @@ -407,8 +408,6 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); - computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } if (mRec->GetParam().rec.tpc.retryRefit) { @@ -427,6 +426,15 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) return mem; } +void* GPUTPCGMMerger::SetPointersRebuild(void* mem) +{ + const size_t nTracks = HostProcessor(this).NMergedTracks(); // Allocated once the number of merged tracks is known on the host + computePointerWithAlignment(mem, mClusterCandidates, nTracks * GPUTPCGeometry::NROWS * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelperRow, nTracks * GPUTPCGeometry::NROWS); + computePointerWithAlignment(mem, mTrackRebuildHelper, nTracks); + return mem; +} + void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) { computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks); @@ -503,6 +511,7 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() mMemoryResOutputO2MC = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersOutputO2MC, GPUMemoryResource::MEMORY_OUTPUT_FLAG | GPUMemoryResource::MEMORY_HOST | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerOutputO2MC"); } } + mMemoryResRebuild = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersRebuild, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerRebuild"); mMemoryResMemory = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT, "TPCMergerMemory"); } @@ -1697,7 +1706,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; + *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; } nHits += nTrackHits; } @@ -2166,6 +2175,16 @@ GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const return Param().tpcMinHitsB5Abs((CAMath::Abs(qPt) + CAMath::Min(CAMath::Sqrt(CAMath::Max(0.f, err2QPt)), Param().rec.tpc.trackFitMinHitsMaxQPtIncrease * CAMath::Abs(qPt))) * Param().qptB5Scaler); } +GPUd() uint32_t GPUTPCGMMerger::CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const +{ + return clusterId - GetConstantMem()->ioPtrs.clustersNative->clusterOffset[sector][row]; +} + +GPUd() uint32_t GPUTPCGMMerger::CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const +{ + return GetConstantMem()->ioPtrs.clustersNative->clusterOffset[candidate.getSector()][row] + candidate.getId(); +} + GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { @@ -2186,10 +2205,10 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, for (uint32_t j = 0; j < trk.NClusters(); j++) { const auto& cl = mClusters[trk.FirstClusterRef() + j]; auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates]; - if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error + if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].isEmpty() || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error for (uint32_t k = 0; k < nMaxCandidates; k++) { - if (candidates[k].id == 0) { - candidates[k] = {.id = cl.num + 2, .row = cl.row, .sector = cl.sector, .error = Param().rec.tpc.rebuildTrackFallbackErr, .weight = 0, .best = 0}; + if (candidates[k].isEmpty()) { + candidates[k].set(CandidateId(cl.num, cl.sector, cl.row), cl.sector, (CAMath::half_t)Param().rec.tpc.rebuildTrackFallbackErr); break; } } @@ -2208,16 +2227,17 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, continue; } for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster - float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2) / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster - int32_t weight = weightf > 0.f ? (weightf > 2e9f ? 2e9f : (int32_t)weightf) : 0; - candidates[iteration].weight = weight; - CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); - CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + auto* c = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + if (helper.getBest() == 0) { + if (c[iteration].hasCluster() && (iteration == 0 || c[iteration].error <= CAMath::Max((CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateGoodErr, c[0].error * (CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2); // Nominal number of clusters and fixed chi2 per cluster for NDF <= 0 + weightf /= (1.f + Param().rec.tpc.rebuildErrorWeightFactor * (float)c[iteration].error); + const int32_t weight = helper.setWeight(weightf); + CAMath::AtomicMax(&mHitWeights[CandidateClusterId(c[iteration], j)], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, CandidateClusterId(c[iteration], j), weight)); } else { - candidates[0].best = -2; + helper.setBest(trackRebuildHelperRow::bestNoCandidate); } } } @@ -2233,10 +2253,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads } for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0) { - if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { - candidates[0].best = iteration + 1; - } + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[iteration], j)]) { + helper.setBest(iteration + 1); } } } @@ -2256,15 +2275,17 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads uint8_t nSharedRowCandidates = 0; for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { - candidates[0].best = nIter; + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0 + if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[nIter - 1], j)]) { + helper.setBest(nIter); } - if (candidates[0].best > 0) { + if (helper.getBest() > 0 && helper.getBest() < trackRebuildHelperRow::bestRowRemoved) { nHits++; - if (candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { - mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations + const uint32_t id = CandidateClusterId(candidates[helper.getBest() - 1], j); + if (helper.getWeight() != mHitWeights[id]) { + mSharedCount2[id] = 1; // shared between tracks of different iterations } - } else if (candidates[0].best != -1 && candidates[0].id >= 2) { + } else if (helper.getBest() != trackRebuildHelperRow::bestRowRemoved && candidates[0].hasCluster()) { sharedRowCandidates[nSharedRowCandidates++] = j; } } @@ -2274,16 +2295,16 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads if (nShared) { const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; - GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [&](const uint8_t& a, const uint8_t& b) { const auto& ca = candidates[a * n]; const auto& cb = candidates[b * n]; - return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? CandidateClusterId(ca, a) < CandidateClusterId(cb, b) : ca.error < cb.error, ca.error < cb.error); }); for (uint32_t j = 0; j < nShared; j++) { auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); - cj[0].best = 1; - mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], CandidateClusterId(cj[0], sharedRowCandidates[j]), cj[0].error)); + mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]].setBest(1); + mSharedCount2[CandidateClusterId(cj[0], sharedRowCandidates[j])] = 1; // TODO: Should we also recover iter0 clusters? } nHits += nShared; } @@ -2304,17 +2325,18 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best > 0) { + if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() > 0 && mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() < trackRebuildHelperRow::bestRowRemoved) { auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; - const auto& best = candidates[candidates[0].best - 1]; - const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + const auto& best = candidates[mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() - 1]; + const uint32_t id = CandidateClusterId(best, j); + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[id]; uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { flags |= GPUTPCGMMergedTrackHit::flagHighIncl; } - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; + outCl = {.num = id, .sector = best.getSector(), .row = (uint8_t)j, .state = flags}; written++; - CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, id)); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 1ca312165f17a..fe3a027799146 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -25,6 +25,7 @@ #include "GPUTPCGMMergerTypes.h" #include "GPUGeneralKernels.h" #include "GPUTPCGeometry.h" +#include "GPUCommonMath.h" #if !defined(GPUCA_GPUCODE) #include @@ -79,20 +80,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { // TODO: Reduce size of this struct! + struct trackCluster { uint32_t id; uint8_t row; uint8_t sector; - float error; - int32_t weight; - int32_t best; }; - struct trackRebuildHelper { - uint8_t reverse; - uint8_t highInclRowLow; - uint8_t highInclRowHigh; - }; + struct trackRebuildHelperCandidate; // Rebuild structs are defined in GPUTPCGMMergerRebuildInternal.h + struct trackRebuildHelper; + struct trackRebuildHelperRow; struct tmpSort { uint32_t x; @@ -113,6 +109,7 @@ class GPUTPCGMMerger : public GPUProcessor void SetMaxData(const GPUTrackingInOutPointers& io); void* SetPointersMerger(void* mem); void* SetPointersRefitScratch(void* mem); + void* SetPointersRebuild(void* mem); void* SetPointersOutput(void* mem); void* SetPointersOutputO2(void* mem); void* SetPointersOutputO2Clus(void* mem); @@ -136,7 +133,8 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } - GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelperCandidate* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelperRow* TrackRebuildHelperRow() { return mTrackRebuildHelperRow; } GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } @@ -173,6 +171,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() uint16_t MemoryResOutputO2Clus() const { return mMemoryResOutputO2Clus; } GPUd() uint16_t MemoryResOutputO2MC() const { return mMemoryResOutputO2MC; } GPUd() uint16_t MemoryResOutputO2Scratch() const { return mMemoryResOutputO2Scratch; } + GPUd() uint16_t MemoryResRebuild() const { return mMemoryResRebuild; } GPUd() int32_t RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack, const GPUTPCTrack* inTrack, float alpha, int32_t sector); GPUd() void SetTrackClusterT(GPUTPCGMSectorTrack& track, int32_t iSector, const GPUTPCTrack* sectorTr); @@ -210,6 +209,8 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() uint32_t CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const; // Id for trackRebuildHelperCandidate (index in the sector row) of a global cluster id + GPUd() uint32_t CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const; // Global cluster id of a candidate with cluster GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); @@ -294,11 +295,13 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2Clus = (uint16_t)-1; uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; + uint16_t mMemoryResRebuild = (uint16_t)-1; - int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) - GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) + GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks o2::MCCompLabel* mMergedTrackMC = nullptr; - trackCluster* mClusterCandidates = nullptr; + trackRebuildHelperCandidate* mClusterCandidates = nullptr; + trackRebuildHelperRow* mTrackRebuildHelperRow = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 2ce927a5a5f7d..1472fa7984068 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -21,6 +21,7 @@ #include "GPUTPCTracker.h" #include "GPUTPCTrackParam.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUO2DataTypes.h" #include "GPUCommonMath.h" #include "GPUTPCTrackParam.h" @@ -273,14 +274,14 @@ void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const out << "Track " << i << ":"; for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].id) { - out << " Row " << j << ": "; - if (candidates[0].best) { - out << " Best " << candidates[0].best << " - "; + if (!candidates[0].isEmpty()) { + out << " Row " << j << " (weight " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getWeight() << "): "; + if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest()) { + out << " Best " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() << " - "; } for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { - if (candidates[k].id) { - out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + if (!candidates[k].isEmpty()) { + out << k << ": id " << (candidates[k].hasCluster() ? CandidateClusterId(candidates[k], j) + 2 : candidates[k].getValue()) << " err " << (float)candidates[k].error << " - "; } } out << "; "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h new file mode 100644 index 0000000000000..ec040469b0658 --- /dev/null +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h @@ -0,0 +1,64 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUTPCGMMergerRebuildInternal.h +/// \author David Rohr + +#ifndef GPUTPCGMMERGERREBUILDINTERNAL_H +#define GPUTPCGMMERGERREBUILDINTERNAL_H + +#include "GPUTPCGMMerger.h" +#include "GPUCommonMath.h" + +namespace o2::gpu +{ + +struct GPUTPCGMMerger::trackRebuildHelperCandidate { // Per track, row and candidate, the row is implicit + uint16_t idHigh, idLow; // 2 x 16 bit instead of uint32_t for a size of 6 bytes without packing, high part first. Combined value: 0: empty (in the first candidate: row not searched), 1: row searched but no candidate, otherwise (sector << 24) | (id + 2), with id the index of the cluster in its sector row (below 2^24 - 2) + CAMath::half_t error; + GPUdi() uint32_t getValue() const { return ((uint32_t)idHigh << 16) | idLow; } + GPUdi() void setValue(uint32_t v) + { + idHigh = v >> 16; + idLow = v & 0xFFFF; + } + GPUdi() bool isEmpty() const { return getValue() == 0; } + GPUdi() bool hasCluster() const { return getValue() >= 2; } + GPUdi() void setNoCandidate() { setValue(1); } + GPUdi() uint32_t getId() const { return (getValue() & 0xFFFFFF) - 2; } // Only with hasCluster(), convert with GPUTPCGMMerger::CandidateId / CandidateClusterId + GPUdi() uint8_t getSector() const { return idHigh >> 8; } // Only with hasCluster() + GPUdi() void set(uint32_t id, uint8_t sector, CAMath::half_t e) + { + setValue(((uint32_t)sector << 24) | (id + 2)); + error = e; + } +}; +static_assert(sizeof(GPUTPCGMMerger::trackRebuildHelperCandidate) == 6); + +struct GPUTPCGMMerger::trackRebuildHelper { // Per track + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; +}; + +struct GPUTPCGMMerger::trackRebuildHelperRow { // Per track and row: weight (lower 28 bits) and best (upper 4 bits) + uint32_t data; + enum : uint32_t { bestRowRemoved = 14, // Row removed (extrapolation reverted) + bestNoCandidate = 15 }; // No acceptable candidate + GPUdi() uint32_t getBest() const { return data >> 28; } // 0: undecided, 1 - 13: index of selected candidate + 1, or bestRowRemoved / bestNoCandidate + GPUdi() void setBest(uint32_t best) { data = (data & 0x0FFFFFFF) | (best << 28); } + GPUdi() int32_t getWeight() const { return data & 0x0FFFFFFF; } + GPUdi() int32_t setWeight(float w) { return (data = (data & 0xF0000000) | (w > 0.5f ? (w < 2147483648.f ? CAMath::Float2UIntReint(w) - 0x3F000000 : 0x0FFFFFFF) : 0)) & 0x0FFFFFFF; } +}; + +} // namespace o2::gpu + +#endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 46254350d1393..0e2a5e94b4291 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -23,6 +23,7 @@ #include "GPUTPCGMMergedTrack.h" #include "GPUTPCGMPolynomialField.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUTPCTracker.h" #include "GPUdEdx.h" #include "GPUParam.h" @@ -385,20 +386,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ inter.markInvalid(); float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - bool found = candidate.id >= 2; + bool found = candidate.hasCluster(); if (found) { lastExtrapolateRow = iExtrapolatedRow; #if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { - int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(merger.CandidateClusterId(candidate, iExtrapolatedRow)), merger.MergedTrackMC()[iTrk]); CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); } #endif float err2Y, err2Z, xx, yy, zz; - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[merger.CandidateClusterId(candidate, iExtrapolatedRow)]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; break; } @@ -417,12 +418,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { revert = true; break; @@ -466,8 +467,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); } } if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped @@ -614,9 +614,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { + if (!candidate.hasCluster() && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation - candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; + candidate.set(merger.CandidateId(cluster.num, cluster.sector, cluster.row), cluster.sector, (CAMath::half_t)(CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])))); } } } else if (!param.rec.tpc.mergerInterpolateErrors && !param.rec.tpc.rebuildTrackInFit) { @@ -798,7 +798,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } int32_t nCandidates = 0; - while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].hasCluster()) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { @@ -854,7 +854,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + if (err < (float)merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { insert = c; break; } @@ -863,17 +863,17 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } // TODO: Downweight non-interpolated candidates - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); } if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].setNoCandidate(); } } else if (doAttach && (!interOnly || inter.isValid())) { const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index ade04a4bdfeb7..dd7d2557f6ebd 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -155,7 +155,7 @@ o2_gpu_kernel_add_parameter(NEIGHBOURS_FINDER_MAX_NNEIGHUP # Number of neighhbo COMP_GATHER_MODE # TPC Compression Gather Mode [0 - 3] SORT_STARTHITS # Sort start hits to improve cache locality during tracklet construction [0/1] CF_SCAN_WORKGROUP_SIZE # Work group size to use in clusterizer scan operation - AMD_EUS_PER_CU) # Number of SIMD units per Compute Unit (only for AMD GPUs) + AMD_EUS_PER_CU) # Number of SIMD units per Compute Unit (only for AMD GPUs) o2_gpu_kernel_add_string_parameter(DEDX_STORAGE_TYPE # Data type to use for intermediate storage of dEdx truncated mean inputs MERGER_INTERPOLATION_ERROR_TYPE) # Data type for storing intermediate track residuals for interpolation From dff9b57acc36f8c6b6109324e0dd6b2b894a9947 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 14:02:29 +0100 Subject: [PATCH 55/60] GPU TPC: Faster implementation for search adjacent clusters in missing rows --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 3 +-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 ++++++++++++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 7dd7d31a4f9cb..510583f01bc93 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -40,8 +40,7 @@ struct InterpolationErrorHit { GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } - GPUdi() bool isAdjacentDone() const { return errorY == (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } - GPUdi() void markAdjacentDone() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Adjacent clusters were searched at the interpolated position of this entry, the entry is invalid afterwards + GPUdi() void markAdjacentSearch() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Invalidates an entry the interpolation rejection did not use, such that the last way searches adjacent clusters for the hit (distinct from the -1 of markInvalid) }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 0e2a5e94b4291..b935ac049c980 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -89,6 +89,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ ConstrainZ(merger, clusters[0].sector, CAMath::Max(t0, tN), CAMath::Min(t0, tN)); } int32_t ihitStart = 0; + int32_t ihitStartInterpolateReject = 0; // First hit of the interpolation rejection way, the hits beyond are not covered by its search for adjacent clusters int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -127,6 +128,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { + ihitStartInterpolateReject = ihitStart; + } for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { if (lastIndexRow != 255) { // One index per pad row (2 if the track has clusters of 2 sectors in the row), counted also for hits skipped below, such that all ways use the same index for a hit interpolationIndex += (clusters[ihit].row == lastIndexRow ? (int32_t)(clusters[ihit].sector != lastIndexSector) : CAMath::Abs(clusters[ihit].row - lastIndexRow)) * wayDirection; @@ -211,6 +215,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissedHard++; NTolerated++; + MarkAdjacentSearch(param, iWay, inter); if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } @@ -224,14 +229,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } + MarkAdjacentSearch(param, iWay, inter); fitBroken = true; break; // bad chi2 for the whole track, stop the fit } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps - } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isAdjacentDone()) : allowChangeClusters)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && (retryAttempt || !inter.isValid() || (ihit - ihitStartInterpolateReject) * wayDirection > 0 || (param.rec.tpc.disableMarkAdjacent & 1))) : allowChangeClusters)) { // With interpolation rejection: in the last way all hits InterpolateReject did not search: in a retry, beyond its first hit, or without interpolated position (also invalidated where it skipped a hit) uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { @@ -249,6 +255,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (currentClusterStatus) { nMissed++; nMissedHard++; + MarkAdjacentSearch(param, iWay, inter); continue; } @@ -286,6 +293,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } // TODO: We can perhaps break here, if we pick up remaining rows } + MarkAdjacentSearch(param, iWay, inter); continue; } } @@ -526,6 +534,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } +GPUdi() void GPUTPCGMTrackParam::MarkAdjacentSearch(const GPUParam& GPUrestrict() param, int32_t iWay, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter) +{ + if (param.rec.tpc.mergerInterpolateErrors && iWay == param.rec.tpc.nWays - 2) { // Hit not passed to InterpolateReject, search adjacent clusters in the last way + inter.markAdjacentSearch(); + } +} + GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { @@ -586,6 +601,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + MarkAdjacentSearch(param, iWay, inter); if (param.rec.tpc.rebuildTrackInFit && (finalOutInFit || iWay < wayRebuildStart + 2)) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds } @@ -726,7 +742,6 @@ GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUre param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); } - inter->markAdjacentDone(); } if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 665c2da4ef14d..31299b2fb8a72 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -160,6 +160,7 @@ class GPUTPCGMTrackParam GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); + GPUd() static void MarkAdjacentSearch(const GPUParam& param, int32_t iWay, gputpcgmmergertypes::InterpolationErrorHit& inter); // For hits the interpolation rejection way skips template GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); From dc6ed486e4ab4e3cb331722eaf8c397a309f77e3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 10:25:02 +0100 Subject: [PATCH 56/60] GPU Config: Fix parsing of empty -- option --- GPU/GPUTracking/utils/qconfig_internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/utils/qconfig_internal.h b/GPU/GPUTracking/utils/qconfig_internal.h index 7e71bcedc226b..057c45db11bdd 100644 --- a/GPU/GPUTracking/utils/qconfig_internal.h +++ b/GPU/GPUTracking/utils/qconfig_internal.h @@ -20,9 +20,9 @@ #ifdef QCONFIG_PARSE -#define QCONFIG_COMPARE(name, optname, optnameshort) \ - (thisoption[0] == '-' && ((thisoption[1] == '-' && optname[0] != 0 && strcmp(&thisoption[2], optname) == 0) || \ - (preoptshort == 0 && thisoption[1] == optnameshort && thisoption[2] == 0) || (thisoption[1] == '-' && strlen(preopt) == 0 && strcmp(&thisoption[2], name) == 0) || \ +#define QCONFIG_COMPARE(name, optname, optnameshort) \ + (thisoption[0] == '-' && ((thisoption[1] == '-' && optname[0] != 0 && strcmp(&thisoption[2], optname) == 0) || \ + (preoptshort == 0 && thisoption[1] == optnameshort && thisoption[2] == 0) || (thisoption[1] == '-' && thisoption[2] != 0 && strlen(preopt) == 0 && strcmp(&thisoption[2], name) == 0) || \ (preoptshort != 0 && thisoption[1] == preoptshort && thisoption[2] == optnameshort && thisoption[3] == 0) || (thisoption[1] == '-' && strlen(preopt) && strncmp(&thisoption[2], preopt, strlen(preopt)) == 0 && strcmp(&thisoption[2 + strlen(preopt)], name) == 0))) #define AddOption(name, type, default, optname, optnameshort, ...) \ From d53e809073ff309b92e8398334efb4f1db2de976 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 09:16:56 +0100 Subject: [PATCH 57/60] GPU TPC: Add TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY feature, to estimate the row range and use atomic prefix-sum to reduce unused row memory for rebuild --- .../Definitions/Parameters/GPUParameters.csv | 1 + .../Parameters/GPUParameters.json.example | 4 + .../Global/GPUChainTrackingMerger.cxx | 17 +++- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 92 ++++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 11 +++ GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 10 +- .../Merger/GPUTPCGMMergerRebuildInternal.h | 50 ++++++++++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 30 +++--- GPU/GPUTracking/kernels.cmake | 25 ++--- 9 files changed, 180 insertions(+), 60 deletions(-) diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index b197c20130738..87336277eadbd 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -109,6 +109,7 @@ GPUTPCConvertKernel,,,,,,,,,,,,,,,256,,,256, PAR:,,,,,,,,,,,,,,,,,,, AMD_EUS_PER_CU,0,0,4,4,,,,,,,,,,,4,,0,4, SORT_STARTHITS,1,0,,,,,,,,1,,1,1,,1,,1,1, +TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY,0,0,,,,,,,,,,,,,,,,, NEIGHBOURS_FINDER_MAX_NNEIGHUP,6,0,10,4,,,,,,4,4,4,4,,5,,2,5, NEIGHBOURS_FINDER_UNROLL_GLOBAL,4,0,4,2,,,,,,2,,8,8,,4,,2,2, NEIGHBOURS_FINDER_UNROLL_SHARED,1,0,0,0,,,,,,1,,1,0,,1,,1,1, diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example index 7390c4b520ae9..f1207e8a6926a 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example @@ -539,6 +539,10 @@ "default": 1, "default_cpu": 0 }, + "TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY": { + "default": 0, + "default_cpu": 0 + }, "NEIGHBOURS_FINDER_MAX_NNEIGHUP": { "default": 6, "default_cpu": 0, diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index b9804a767ad2e..634520e2f90a1 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -207,9 +207,14 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } + const bool rebuildConstrainedRows = param().rec.tpc.rebuildTrackInFit && mRec->getGPUParameters(doGPU).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY; if (param().rec.tpc.rebuildTrackInFit) { mRec->PushNonPersistentMemory(qStr2Tag("TPCREBLD")); // The rebuild memory is freed after the fit, before the O2 output } + if (rebuildConstrainedRows) { + AllocateRegisteredMemory(Merger.MemoryResRebuild2()); + WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); + } runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); @@ -217,6 +222,11 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel(GetGridAuto(0, deviceType)); runKernel(GetGridAuto(0, deviceType)); runKernel(GetGridAuto(0, deviceType)); + if (doGPU && rebuildConstrainedRows) { // PrepareForFit1 computed nRebuildTrackRow, needed on the host to allocate the rebuild memory, waited for below + ReleaseEvent(mEvents->single); + TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0, &mEvents->single); + waitForTransfer = 1; + } DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingPrepareFit, doGPU, Merger, &GPUTPCGMMerger::DumpFitPrepare, *mDebugFile); @@ -237,10 +247,11 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (param().rec.tpc.rebuildTrackInFit) { - AllocateRegisteredMemory(Merger.MemoryResRebuild()); // Sized by the number of merged tracks, which was transferred to the host above + AllocateRegisteredMemory(Merger.MemoryResRebuild()); // With constrained rows sized by nRebuildTrackRow, which was transferred to the host above WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * sizeof(*MergerShadow.TrackRebuildHelperRow())); + const size_t nRebuildRows = rebuildConstrainedRows ? (size_t)Merger.Memory()->nRebuildTrackRow : (size_t)Merger.NMergedTracks() * GPUTPCGeometry::NROWS; + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), nRebuildRows * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), nRebuildRows * sizeof(*MergerShadow.TrackRebuildHelperRow())); } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 26d83fdfa77a3..6393aa82c28fe 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -428,10 +428,20 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) void* GPUTPCGMMerger::SetPointersRebuild(void* mem) { - const size_t nTracks = HostProcessor(this).NMergedTracks(); // Allocated once the number of merged tracks is known on the host - computePointerWithAlignment(mem, mClusterCandidates, nTracks * GPUTPCGeometry::NROWS * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); - computePointerWithAlignment(mem, mTrackRebuildHelperRow, nTracks * GPUTPCGeometry::NROWS); - computePointerWithAlignment(mem, mTrackRebuildHelper, nTracks); + const bool constrained = mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY; + const uint64_t nRows = constrained ? (uint64_t)HostProcessor(this).mMemory->nRebuildTrackRow : (uint64_t)HostProcessor(this).NMergedTracks() * GPUTPCGeometry::NROWS; + if (nRows * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates > 0xFFFFFFFFu) { + throw std::bad_alloc(); + } + computePointerWithAlignment(mem, mClusterCandidates, nRows * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelperRow, nRows); + computePointerWithAlignment(mem, mTrackRebuildHelper, HostProcessor(this).NMergedTracks()); + return mem; +} + +void* GPUTPCGMMerger::SetPointersRebuild2(void* mem) +{ + computePointerWithAlignment(mem, mTrackRebuildHelperRowInfo, mNMaxTracks); // Needed before the row ranges are known, only with par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY return mem; } @@ -512,6 +522,7 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() } } mMemoryResRebuild = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersRebuild, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerRebuild"); + mMemoryResRebuild2 = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersRebuild2, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerRebuild2"); mMemoryResMemory = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT, "TPCMergerMemory"); } @@ -1860,6 +1871,9 @@ GPUd() void GPUTPCGMMerger::PrepareForFit0(int32_t nBlocks, int32_t nThreads, in for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTracks; i += nBlocks * nThreads) { mTrackSort[i] = i; } + if (iThread == 0 && iBlock == 0) { + mMemory->nRebuildTrackRow = 0; + } } GPUd() void GPUTPCGMMerger::SortTracks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) @@ -1887,12 +1901,39 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { const bool protect = CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0; + [[maybe_unused]] uint8_t rowMin = 255, rowMax = 0; for (uint32_t j = 0; j < trk.NClusters(); j++) { + rowMin = CAMath::Min(rowMin, mClusters[trk.FirstClusterRef() + j].row); + rowMax = CAMath::Max(rowMax, mClusters[trk.FirstClusterRef() + j].row); if (protect) { // Protect the clusters of the initial tracks. No track id on purpose, any entry of a final track for the cluster shall win CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], (uint32_t)(attachAttached | attachProtect | (Param().rec.tpc.rebuildTrackInFit ? attachZero : attachGood))); } CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + if (Param().rec.tpc.rebuildTrackInFit) { // Row range the rebuild can use: cluster rows and the radii the helix can reach, plus margin + const auto& GPUrestrict() p = trk.GetParam(); + const float k = -p.GetQPt() * Param().bzCLight; + int32_t rowFirst = 0, rowLast = GPUTPCGeometry::NROWS - 1; + if (CAMath::Abs(k) > 1e-4f && p.GetX() > 50.f && p.GetX() < 300.f && CAMath::Abs(p.GetY()) < p.GetX()) { // Otherwise (almost straight tracks, implausible seed) full range + const float cosPhi = CAMath::Sqrt(CAMath::Max(0.f, 1.f - p.GetSinPhi() * p.GetSinPhi())); + const float r1 = 1.f / k, r2 = 2.f / k; // Signed radius for nominal pt, and for twice the pt as upper limit of the circle + const float f1 = CAMath::Sqrt(CAMath::Square(p.GetX() - p.GetSinPhi() * r1) + CAMath::Square(p.GetY() + cosPhi * r1)) - CAMath::Abs(r1); // Distance of the circle center from the beam axis minus radius + const float c2 = CAMath::Sqrt(CAMath::Square(p.GetX() - p.GetSinPhi() * r2) + CAMath::Square(p.GetY() + cosPhi * r2)), f2 = c2 - CAMath::Abs(r2); + const float rInner = (f1 > 0) != (f2 > 0) ? 0.f : (CAMath::Min(CAMath::Abs(f1), CAMath::Abs(f2)) * 0.9848f), rOuter = c2 + CAMath::Abs(r2); // Inner radius can be smaller for lower pt if the beam axis is inside the circle. 0.9848 = cos(10 deg): local x at the sector edge + while (rowFirst < (int32_t)GPUTPCGeometry::NROWS - 1 && GPUTPCGeometry::Row2X(rowFirst) < rInner) { + rowFirst++; + } + while (rowLast > 0 && GPUTPCGeometry::Row2X(rowLast) > rOuter) { + rowLast--; + } + } + auto& GPUrestrict() helper = mTrackRebuildHelperRowInfo[i]; + helper.firstRow = CAMath::Max(0, CAMath::Min(rowFirst, rowMin) - 5); + helper.lastRow = CAMath::Min(GPUTPCGeometry::NROWS - 1, CAMath::Max(rowLast, rowMax) + 5); + helper.firstRowIndex = CAMath::AtomicAdd(&mMemory->nRebuildTrackRow, (uint32_t)(helper.lastRow - helper.firstRow + 1)); + } + } if (!trk.CCE() && !trk.MergedLooper()) { GPUTPCGMMergedTrack* updTrk = trk.GetFirstSegment(mMergedTracks, Param().rec.enableCyclicGraphWorkarounds); const auto &cl0 = mClusters[trk.FirstClusterRef()], &cln = mClusters[updTrk->FirstClusterRef() + updTrk->NClusters() - 1]; @@ -2204,7 +2245,10 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int lastRow = -1; for (uint32_t j = 0; j < trk.NClusters(); j++) { const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates]; + if (!RebuildRowInRange(i, cl.row)) { + continue; + } + auto* candidates = ClusterCandidates(i, cl.row); if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].isEmpty() || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error for (uint32_t k = 0; k < nMaxCandidates; k++) { if (candidates[k].isEmpty()) { @@ -2226,9 +2270,9 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, if (!trk.OK()) { continue; } - for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* c = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + for (uint32_t j = RebuildFirstRow(i), jLast = RebuildLastRow(i); j <= jLast; j++) { + auto* c = ClusterCandidates(i, j); + auto& helper = TrackRebuildHelperRow(i, j); if (helper.getBest() == 0) { if (c[iteration].hasCluster() && (iteration == 0 || c[iteration].error <= CAMath::Max((CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateGoodErr, c[0].error * (CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2); // Nominal number of clusters and fixed chi2 per cluster for NDF <= 0 @@ -2251,9 +2295,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads if (!trk.OK()) { continue; } - for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + for (uint32_t j = RebuildFirstRow(i), jLast = RebuildLastRow(i); j <= jLast; j++) { + auto* candidates = ClusterCandidates(i, j); + auto& helper = TrackRebuildHelperRow(i, j); if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[iteration], j)]) { helper.setBest(iteration + 1); } @@ -2273,9 +2317,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads uint32_t nHits = 0; uint8_t sharedRowCandidates[GPUTPCGeometry::NROWS]; uint8_t nSharedRowCandidates = 0; - for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0 + for (uint32_t j = RebuildFirstRow(i), jLast = RebuildLastRow(i); j <= jLast; j++) { + auto* candidates = ClusterCandidates(i, j); + auto& helper = TrackRebuildHelperRow(i, j); // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0 if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[nIter - 1], j)]) { helper.setBest(nIter); } @@ -2293,17 +2337,15 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads if (nSharedRowCandidates) { const uint32_t nShared = CAMath::Min(nSharedRowCandidates, CAMath::Float2UIntRn(nHits * Param().rec.tpc.rebuildTrackMaxSharedFraction)); if (nShared) { - const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; - const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; - GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [&](const uint8_t& a, const uint8_t& b) { - const auto& ca = candidates[a * n]; - const auto& cb = candidates[b * n]; + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [i, this](const uint8_t& a, const uint8_t& b) { + const auto& ca = ClusterCandidates(i, a)[0]; + const auto& cb = ClusterCandidates(i, b)[0]; return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? CandidateClusterId(ca, a) < CandidateClusterId(cb, b) : ca.error < cb.error, ca.error < cb.error); }); for (uint32_t j = 0; j < nShared; j++) { - auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + auto* cj = ClusterCandidates(i, sharedRowCandidates[j]); CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], CandidateClusterId(cj[0], sharedRowCandidates[j]), cj[0].error)); - mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]].setBest(1); + TrackRebuildHelperRow(i, sharedRowCandidates[j]).setBest(1); mSharedCount2[CandidateClusterId(cj[0], sharedRowCandidates[j])] = 1; // TODO: Should we also recover iter0 clusters? } nHits += nShared; @@ -2323,11 +2365,11 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads trk.SetFirstClusterRef(iOut); uint32_t written = 0; const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut - for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { - const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() > 0 && mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() < trackRebuildHelperRow::bestRowRemoved) { + for (uint32_t j = RebuildFirstRow(i), jLast = RebuildLastRow(i); j <= jLast; j++) { + const auto& candidates = ClusterCandidates(i, j); + if (TrackRebuildHelperRow(i, j).getBest() > 0 && TrackRebuildHelperRow(i, j).getBest() < trackRebuildHelperRow::bestRowRemoved) { auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; - const auto& best = candidates[mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() - 1]; + const auto& best = candidates[TrackRebuildHelperRow(i, j).getBest() - 1]; const uint32_t id = CandidateClusterId(best, j); const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[id]; uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index fe3a027799146..d483f34984fc9 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -73,6 +73,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; GPUAtomic(uint32_t) nMergedTrackClusters; + GPUAtomic(uint32_t) nRebuildTrackRow; // Sum of the row ranges of all tracks for the rebuild GPUAtomic(uint32_t) nO2Tracks; GPUAtomic(uint32_t) nO2ClusRefs; const GPUTPCTrack* firstExtrapolatedTracks[NSECTORS]; @@ -88,6 +89,7 @@ class GPUTPCGMMerger : public GPUProcessor struct trackRebuildHelperCandidate; // Rebuild structs are defined in GPUTPCGMMergerRebuildInternal.h struct trackRebuildHelper; + struct trackRebuildHelperRowInfo; struct trackRebuildHelperRow; struct tmpSort { @@ -110,6 +112,7 @@ class GPUTPCGMMerger : public GPUProcessor void* SetPointersMerger(void* mem); void* SetPointersRefitScratch(void* mem); void* SetPointersRebuild(void* mem); + void* SetPointersRebuild2(void* mem); void* SetPointersOutput(void* mem); void* SetPointersOutputO2(void* mem); void* SetPointersOutputO2Clus(void* mem); @@ -135,6 +138,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } GPUhdi() trackRebuildHelperCandidate* ClusterCandidates() { return mClusterCandidates; } GPUhdi() trackRebuildHelperRow* TrackRebuildHelperRow() { return mTrackRebuildHelperRow; } + GPUdi() trackRebuildHelperCandidate* ClusterCandidates(int32_t iTrk, uint32_t row) const; + GPUdi() trackRebuildHelperRow& TrackRebuildHelperRow(int32_t iTrk, uint32_t row) const; + GPUdi() bool RebuildRowInRange(int32_t iTrk, uint32_t row) const; + GPUdi() uint32_t RebuildFirstRow(int32_t iTrk) const; // Row range of the track for the rebuild, all rows without GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY + GPUdi() uint32_t RebuildLastRow(int32_t iTrk) const; GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } @@ -172,6 +180,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() uint16_t MemoryResOutputO2MC() const { return mMemoryResOutputO2MC; } GPUd() uint16_t MemoryResOutputO2Scratch() const { return mMemoryResOutputO2Scratch; } GPUd() uint16_t MemoryResRebuild() const { return mMemoryResRebuild; } + GPUd() uint16_t MemoryResRebuild2() const { return mMemoryResRebuild2; } GPUd() int32_t RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack, const GPUTPCTrack* inTrack, float alpha, int32_t sector); GPUd() void SetTrackClusterT(GPUTPCGMSectorTrack& track, int32_t iSector, const GPUTPCTrack* sectorTr); @@ -296,6 +305,7 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; uint16_t mMemoryResRebuild = (uint16_t)-1; + uint16_t mMemoryResRebuild2 = (uint16_t)-1; int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks @@ -303,6 +313,7 @@ class GPUTPCGMMerger : public GPUProcessor trackRebuildHelperCandidate* mClusterCandidates = nullptr; trackRebuildHelperRow* mTrackRebuildHelperRow = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; + trackRebuildHelperRowInfo* mTrackRebuildHelperRowInfo = nullptr; int32_t* mHitWeights = nullptr; GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 1472fa7984068..cabe3eb7db923 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -272,12 +272,12 @@ void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const const auto& trk = mMergedTracks[i]; if (trk.OK() && trk.GetParam().GetNDF() >= 0 && trk.NClusters()) { out << "Track " << i << ":"; - for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + for (uint32_t j = RebuildFirstRow(i), jLast = RebuildLastRow(i); j <= jLast; j++) { + auto* candidates = ClusterCandidates(i, j); if (!candidates[0].isEmpty()) { - out << " Row " << j << " (weight " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getWeight() << "): "; - if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest()) { - out << " Best " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() << " - "; + out << " Row " << j << " (weight " << TrackRebuildHelperRow(i, j).getWeight() << "): "; + if (TrackRebuildHelperRow(i, j).getBest()) { + out << " Best " << TrackRebuildHelperRow(i, j).getBest() << " - "; } for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { if (!candidates[k].isEmpty()) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h index ec040469b0658..1d4cc8de0c454 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h @@ -43,6 +43,11 @@ struct GPUTPCGMMerger::trackRebuildHelperCandidate { // Per track, row and candi }; static_assert(sizeof(GPUTPCGMMerger::trackRebuildHelperCandidate) == 6); +struct __attribute__((packed)) GPUTPCGMMerger::trackRebuildHelperRowInfo { // Per track, only with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY + uint32_t firstRowIndex; // Index of firstRow of the track in mTrackRebuildHelperRow and (times the number of candidates) in mClusterCandidates + uint8_t firstRow, lastRow; // Row range the rebuild can use for the track, set in PrepareForFit1 +}; + struct GPUTPCGMMerger::trackRebuildHelper { // Per track uint8_t reverse; uint8_t highInclRowLow; @@ -59,6 +64,51 @@ struct GPUTPCGMMerger::trackRebuildHelperRow { // Per track and row: weight (low GPUdi() int32_t setWeight(float w) { return (data = (data & 0xF0000000) | (w > 0.5f ? (w < 2147483648.f ? CAMath::Float2UIntReint(w) - 0x3F000000 : 0x0FFFFFFF) : 0)) & 0x0FFFFFFF; } }; +GPUdi() uint32_t GPUTPCGMMerger::RebuildFirstRow(int32_t iTrk) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + return mTrackRebuildHelperRowInfo[iTrk].firstRow; + } else { + return 0; + } +} + +GPUdi() uint32_t GPUTPCGMMerger::RebuildLastRow(int32_t iTrk) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + return mTrackRebuildHelperRowInfo[iTrk].lastRow; + } else { + return GPUTPCGeometry::NROWS - 1; + } +} + +GPUdi() bool GPUTPCGMMerger::RebuildRowInRange(int32_t iTrk, uint32_t row) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + return row - mTrackRebuildHelperRowInfo[iTrk].firstRow <= (uint32_t)(mTrackRebuildHelperRowInfo[iTrk].lastRow - mTrackRebuildHelperRowInfo[iTrk].firstRow); + } else { + return true; // No guard needed, the memory covers all rows + } +} + +GPUdi() GPUTPCGMMerger::trackRebuildHelperCandidate* GPUTPCGMMerger::ClusterCandidates(int32_t iTrk, uint32_t row) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + return &mClusterCandidates[(mTrackRebuildHelperRowInfo[iTrk].firstRowIndex + row - mTrackRebuildHelperRowInfo[iTrk].firstRow) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + } else { + return &mClusterCandidates[(iTrk * GPUTPCGeometry::NROWS + row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + } +} + +GPUdi() GPUTPCGMMerger::trackRebuildHelperRow& GPUTPCGMMerger::TrackRebuildHelperRow(int32_t iTrk, uint32_t row) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + return mTrackRebuildHelperRow[mTrackRebuildHelperRowInfo[iTrk].firstRowIndex + row - mTrackRebuildHelperRowInfo[iTrk].firstRow]; + } else { + return mTrackRebuildHelperRow[iTrk * GPUTPCGeometry::NROWS + row]; + } +} + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index b935ac049c980..9e5cdbdcc139a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -348,7 +348,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ bool revert = false; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; - for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + for (; iExtrapolatedRow < GPUTPCGeometry::NROWS && merger.RebuildRowInRange(iTrk, iExtrapolatedRow); iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers bool fail = false; for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; @@ -393,7 +393,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + auto& candidate = merger.ClusterCandidates(iTrk, iExtrapolatedRow)[0]; bool found = candidate.hasCluster(); if (found) { lastExtrapolateRow = iExtrapolatedRow; @@ -407,7 +407,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[merger.CandidateClusterId(candidate, iExtrapolatedRow)]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { - merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; break; } @@ -426,12 +426,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else - merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { - merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { revert = true; break; @@ -475,7 +475,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { - merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); } } if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped @@ -628,8 +628,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger rejectChi2 = true; } } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { - if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; + if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid() && merger.RebuildRowInRange(iTrk, cluster.row)) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates(iTrk, cluster.row)[0]; if (!candidate.hasCluster() && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation candidate.set(merger.CandidateId(cluster.num, cluster.sector, cluster.row), cluster.sector, (CAMath::half_t)(CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])))); @@ -764,7 +764,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; } const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; - if (useInter || useCurrent) { + if ((useInter || useCurrent) && merger.RebuildRowInRange(iTrk, row)) { // No candidates for rows outside of the row range of the track const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -813,7 +813,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } int32_t nCandidates = 0; - while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].hasCluster()) { + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates(iTrk, row)[nCandidates].hasCluster()) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { @@ -869,26 +869,26 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { - if (err < (float)merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + if (err < (float)merger.ClusterCandidates(iTrk, row)[c].error) { insert = c; break; } } if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + merger.ClusterCandidates(iTrk, row)[c] = merger.ClusterCandidates(iTrk, row)[c - 1]; } // TODO: Downweight non-interpolated candidates - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); + merger.ClusterCandidates(iTrk, row)[insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = merger.ClusterCandidates(iTrk, row); for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); } if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].setNoCandidate(); + merger.ClusterCandidates(iTrk, row)[0].setNoCandidate(); } } else if (doAttach && (!interOnly || inter.isValid())) { const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index dd7d2557f6ebd..45e9f30add615 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -144,18 +144,19 @@ o2_gpu_add_kernel("GPUTPCNNClusterizerKernels, publishClass2Regression" "= TPC o2_gpu_add_kernel("GPUTPCNNClusterizerKernels, publishDeconvolutionFlags" "= TPCNNCLUSTERFINDER" LB uint8_t sector int8_t dtype int8_t withMC uint32_t batchStart) endif() -o2_gpu_kernel_add_parameter(NEIGHBOURS_FINDER_MAX_NNEIGHUP # Number of neighhbours finder hits to cache in shared memory - NEIGHBOURS_FINDER_UNROLL_GLOBAL # Unroll factor for neighbours finder iterating hits in local memory - NEIGHBOURS_FINDER_UNROLL_SHARED # Fully unroll iteration over neighbours finder hits in shared memory [0/1] - TRACKLET_SELECTOR_HITS_REG_SIZE # Number of hits to cache in shared memory in tracklet selector - ALTERNATE_BORDER_SORT # Use alternative border sort approach [0/1] - SORT_BEFORE_FIT # Sort tracks after length to reduce warp serialization [0/1] - NO_ATOMIC_PRECHECK # Skip atomic precheck to reduce posterior synchronization [0/1] - COMP_GATHER_KERNEL # Default kernel to use for Compression Gather Operation [0 - 4] - COMP_GATHER_MODE # TPC Compression Gather Mode [0 - 3] - SORT_STARTHITS # Sort start hits to improve cache locality during tracklet construction [0/1] - CF_SCAN_WORKGROUP_SIZE # Work group size to use in clusterizer scan operation - AMD_EUS_PER_CU) # Number of SIMD units per Compute Unit (only for AMD GPUs) +o2_gpu_kernel_add_parameter(NEIGHBOURS_FINDER_MAX_NNEIGHUP # Number of neighhbours finder hits to cache in shared memory + NEIGHBOURS_FINDER_UNROLL_GLOBAL # Unroll factor for neighbours finder iterating hits in local memory + NEIGHBOURS_FINDER_UNROLL_SHARED # Fully unroll iteration over neighbours finder hits in shared memory [0/1] + TRACKLET_SELECTOR_HITS_REG_SIZE # Number of hits to cache in shared memory in tracklet selector + ALTERNATE_BORDER_SORT # Use alternative border sort approach [0/1] + SORT_BEFORE_FIT # Sort tracks after length to reduce warp serialization [0/1] + NO_ATOMIC_PRECHECK # Skip atomic precheck to reduce posterior synchronization [0/1] + COMP_GATHER_KERNEL # Default kernel to use for Compression Gather Operation [0 - 4] + COMP_GATHER_MODE # TPC Compression Gather Mode [0 - 3] + SORT_STARTHITS # Sort start hits to improve cache locality during tracklet construction [0/1] + TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY # Allocate the track rebuild memory only for the row range a track can reach [0/1] + CF_SCAN_WORKGROUP_SIZE # Work group size to use in clusterizer scan operation + AMD_EUS_PER_CU) # Number of SIMD units per Compute Unit (only for AMD GPUs) o2_gpu_kernel_add_string_parameter(DEDX_STORAGE_TYPE # Data type to use for intermediate storage of dEdx truncated mean inputs MERGER_INTERPOLATION_ERROR_TYPE) # Data type for storing intermediate track residuals for interpolation From 2e224a3e9fbb7b0cdedfb5e4ce4e0319dc09693f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 10:00:33 +0100 Subject: [PATCH 58/60] GPU RTC: Add overrideRTCParameters option --- .../Base/GPUReconstructionRTCHelpers.inc | 40 +++++++----- .../GPUDefParametersLoad.template.inc | 63 +++++++++++++++++++ GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionRTCHelpers.inc b/GPU/GPUTracking/Base/GPUReconstructionRTCHelpers.inc index f6f794315a783..a09dacc874ed0 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionRTCHelpers.inc +++ b/GPU/GPUTracking/Base/GPUReconstructionRTCHelpers.inc @@ -69,23 +69,32 @@ static std::string baseCommand(const GPUSettingsProcessing& proc, std::string_vi static void loadLaunchBounds(const GPUSettingsProcessing& proc, GPUDefParameters& par) { - if (proc.rtctech.loadLaunchBoundsFromFile.size() == 0) { - return; - } - FILE* fp = fopen(proc.rtctech.loadLaunchBoundsFromFile.c_str(), "rb"); - if (fp == nullptr) { - throw std::runtime_error("Cannot open launch bounds parameter module file"); - } - fseek(fp, 0, SEEK_END); - size_t size = ftell(fp); - if (size != sizeof(par)) { - throw std::runtime_error("launch bounds parameter file has incorrect size"); + if (proc.rtctech.loadLaunchBoundsFromFile.size()) { + FILE* fp = fopen(proc.rtctech.loadLaunchBoundsFromFile.c_str(), "rb"); + if (fp == nullptr) { + throw std::runtime_error("Cannot open launch bounds parameter module file"); + } + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + if (size != sizeof(par)) { + throw std::runtime_error("launch bounds parameter file has incorrect size"); + } + fseek(fp, 0, SEEK_SET); + if (fread(&par, 1, size, fp) != size) { + throw std::runtime_error("Error reading launch bounds parameter file"); + } + fclose(fp); } - fseek(fp, 0, SEEK_SET); - if (fread(&par, 1, size, fp) != size) { - throw std::runtime_error("Error reading launch bounds parameter file"); + std::vector overrides; // Split at ;, trim spaces, skip empty entries + const std::string& str = proc.rtctech.overrideRTCParameters; + for (size_t pos = 0; pos < str.size();) { + const size_t end = std::min(str.find(';', pos), str.size()), first = str.find_first_not_of(' ', pos); + if (first < end) { + overrides.emplace_back(str.substr(first, str.find_last_not_of(' ', end - 1) - first + 1)); + } + pos = end + 1; } - fclose(fp); + o2::gpu::internal::GPUDefParametersOverride(par, overrides); } static std::string launchBoundsCode(const GPUSettingsProcessing& proc, const GPUDefParameters& par, int32_t minBlockFactor, uint32_t warpSize) // Prints the launch bounds if requested @@ -93,6 +102,7 @@ static std::string launchBoundsCode(const GPUSettingsProcessing& proc, const GPU std::string retVal = GPUDefParametersExport(par, true, minBlockFactor) + "#define GPUCA_WARP_SIZE " + std::to_string(warpSize) + "\n"; if (proc.rtctech.printLaunchBounds || proc.debugLevel >= 3) { GPUInfo("RTC Launch Bounds:\n%s", retVal.c_str()); + fflush(stdout); // printLaunchBounds == 2 ends the process via an exception, which would lose buffered output } return retVal; } diff --git a/GPU/GPUTracking/Definitions/GPUDefParametersLoad.template.inc b/GPU/GPUTracking/Definitions/GPUDefParametersLoad.template.inc index 8b7a79a9e48bf..074d7836c38d6 100644 --- a/GPU/GPUTracking/Definitions/GPUDefParametersLoad.template.inc +++ b/GPU/GPUTracking/Definitions/GPUDefParametersLoad.template.inc @@ -22,6 +22,9 @@ #include "GPUDefMacros.h" #include #include +#include +#include +#include namespace o2::gpu::internal { @@ -68,4 +71,64 @@ static std::string GPUDefParametersExport(const GPUDefParameters& par, bool forR return o.str(); // clang-format on } +#define GPUCA_OVERRIDE_KERNEL_LB(name) \ + if (key == GPUCA_M_STR(name)) { \ + par.par_LB_maxThreads[i] = par.par_LB_minBlocks[i] = par.par_LB_forceBlocks[i] = 0; \ + std::stringstream v(val.front() == '[' && val.back() == ']' ? val.substr(1, val.size() - 2) : val); \ + std::string tok; \ + for (int32_t j = 0; std::getline(v, tok, ','); j++) { \ + if (j >= 3) { \ + throw std::runtime_error("Too many values in launch bounds override " + entry); \ + } \ + (j == 0 ? par.par_LB_maxThreads : (j == 1 ? par.par_LB_minBlocks : par.par_LB_forceBlocks))[i] = std::stoi(tok); \ + } \ + found = true; \ + } \ + i++; + +#define GPUCA_OVERRIDE_KERNEL_PARAM(name) \ + if (key == GPUCA_M_STR(name)) { \ + GPUCA_M_CAT(par.par_, name) = std::stoi(val); \ + found = true; \ + } + +#define GPUCA_OVERRIDE_KERNEL_STRING_PARAM(name) \ + if (key == GPUCA_M_STR(name)) { \ + if (val.size() >= sizeof(GPUCA_M_CAT(par.par_, name))) { \ + throw std::runtime_error("Value too long in parameter override " + entry); \ + } \ + strcpy(GPUCA_M_CAT(par.par_, name), val.c_str()); \ + found = true; \ + } + +static void GPUDefParametersOverride(GPUDefParameters& par, const std::vector& overrides) // Entries name=value: name is a kernel name as in the parameter file for launch bounds (value n or [maxThreads, minBlocks, forceBlocks], values not given are reset), or the name of a parameter +{ + const auto trim = [](const std::string& str) { + const size_t first = str.find_first_not_of(' '), last = str.find_last_not_of(' '); + return first == std::string::npos ? std::string() : str.substr(first, last - first + 1); + }; + for (const std::string& entry : overrides) { + const size_t eq = entry.find('='); + const std::string key = trim(entry.substr(0, eq)), val = eq == std::string::npos ? std::string() : trim(entry.substr(eq + 1)); + if (key.empty() || val.empty()) { + throw std::runtime_error("Invalid parameter override " + entry); + } + bool found = false; + int32_t i = 0; + try { // clang-format off + $,REPLACE,[^A-Za-z0-9]+,_>,PREPEND,GPUCA_OVERRIDE_KERNEL_LB(>,APPEND,)>, + > + $,PREPEND,GPUCA_OVERRIDE_KERNEL_PARAM(>,APPEND,)>, + > + $,PREPEND,GPUCA_OVERRIDE_KERNEL_STRING_PARAM(>,APPEND,)>, + > + } catch (const std::logic_error&) { // clang-format on + throw std::runtime_error("Invalid value in parameter override " + entry); + } + if (!found) { + throw std::runtime_error("Unknown parameter in override " + entry); + } + } +} + } // namespace o2::gpu::internal diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 56d7b004c9212..7350ff19b18eb 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -266,6 +266,7 @@ AddOption(cacheSize, uint32_t, 10, "", 0, "Maximum number of RTC cache files per AddOption(prependCommand, std::string, "", "", 0, "Prepend RTC compilation commands by this string") AddOption(overrideArchitecture, std::string, "", "", 0, "Override arhcitecture part of RTC compilation command line") // Part of cmdLine, so checked against the cache AddOption(loadLaunchBoundsFromFile, std::string, "", "", 0, "Load a parameter object containing the launch bounds from a file") +AddOption(overrideRTCParameters, std::string, "", "", 0, "Semicolon-separated list of overrides of launch bounds and parameters for RTC compilation: 'kernel=64;kernel2=[64,4];par1=1;par2=foo'") AddOption(keepTempFiles, bool, false, "", 0, "Keep temporary source and object files") AddHelp("help", 'h') EndConfig() From 38f7da1d286f0cd17fff84d163014f7714b0caf5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 11:06:27 +0100 Subject: [PATCH 59/60] GPU Config: Refactor: use hashmap of handlers instead of many ifs with string comparison --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 24 ++-- .../Definitions/GPUSettingsListDisplay.h | 12 +- .../Definitions/GPUSettingsListStandalone.h | 6 +- GPU/GPUTracking/utils/qconfig.cxx | 113 +++++++++++++-- GPU/GPUTracking/utils/qconfig_internal.h | 131 +++++------------- 5 files changed, 161 insertions(+), 125 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 7350ff19b18eb..91d0856718933 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -191,7 +191,7 @@ AddOptionRTC(rebuildTrackFallbackMaxNCl, uint8_t, 20, "", 0, "For tracks with at AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddVariableRTC(nWays, uint8_t, 0) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Reconstruction parameters for TRD, no bool in here !!! @@ -216,12 +216,12 @@ AddOptionRTC(matCorrType, uint8_t, 2, "", 0, "Material correction to use: 0 - no AddOptionRTC(pileupFwdNBC, uint8_t, 70, "", 0, "Post-trigger Pile-up integration time in BCs") AddOptionRTC(pileupBwdNBC, uint8_t, 130, "", 0, "Pre-trigger Pile-up integration time in BCs") AddOptionRTC(useAngularPull, uint8_t, 3, "", 0, "0 = don't use angular pull; 1 = additional error based on angular pull for refit only; 2 = add error also for chi2; 3 and 4 = correction of the average position based on angular pull") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Dynamic reconstruction parameters, no bool in here!!!, must NOT use AddOptionRTC(...) !!! BeginSubConfig(GPUSettingsRecDynamic, dyn, configStandalone.rec, "RECDYN", 0, "Reconstruction settings", rec_dyn) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Global reconstruction parameters, no bool in here !!! @@ -238,7 +238,7 @@ AddCustomCPP(void SetMinTrackPtB5(float v) { maxTrackQPtB5 = v > 0.001f ? (1.f / AddSubConfig(GPUSettingsRecTPC, tpc) AddSubConfig(GPUSettingsRecTRD, trd) AddSubConfig(GPUSettingsRecDynamic, dyn) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() #if !defined(__OPENCL__) && !defined(__METAL__) // these use std::string / std::vector, which device code does not have @@ -251,7 +251,7 @@ AddOption(deterministic, bool, false, "", 0, "Compile RTC in deterministic mode, AddOption(compilePerKernel, bool, true, "", 0, "Run one RTC compilation per kernel") AddOption(enable, bool, false, "", 0, "Use RTC to optimize GPU code") AddOption(overrideWarpSize, int32_t, -1, "", 0, "Override the warp size to be used for RTC") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Technical parameters for RunTimeCompilation, which do not change the RTC code @@ -268,7 +268,7 @@ AddOption(overrideArchitecture, std::string, "", "", 0, "Override arhcitecture p AddOption(loadLaunchBoundsFromFile, std::string, "", "", 0, "Load a parameter object containing the launch bounds from a file") AddOption(overrideRTCParameters, std::string, "", "", 0, "Semicolon-separated list of overrides of launch bounds and parameters for RTC compilation: 'kernel=64;kernel2=[64,4];par1=1;par2=foo'") AddOption(keepTempFiles, bool, false, "", 0, "Keep temporary source and object files") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Parameters that steer reconstruction that do not go to the device, or only in derrived form. @@ -276,7 +276,7 @@ BeginSubConfig(GPUSettingsProcessingParam, param, configStandalone.proc, "PARAM" AddOptionArray(tpcErrorParamY, float, 4, (0.06f, 0.24f, 0.12f, 0.1f), "", 0, "TPC Cluster Y Error Parameterization") AddOptionArray(tpcErrorParamZ, float, 4, (0.06f, 0.24f, 0.15f, 0.1f), "", 0, "TPC Cluster Z Error Parameterization") AddOption(tpcTriggerHandling, bool, true, "", 0, "Enable TPC trigger handling") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings steering the processing of NN Clusterization @@ -323,7 +323,7 @@ AddOption(nnCCDBRegressionLayerType, std::string, "FC", "", 0, "Distinguishes be AddOption(nnCCDBBeamType, std::string, "pp", "", 0, "Distinguishes between networks trained for different beam types. Options: pp, pPb, PbPb") AddOption(nnCCDBInteractionRate, std::string, "500", "", 0, "Distinguishes between networks for different interaction rates [kHz]") AddOption(nnCCDBExtraMetadata, std::string, "", "", 0, "Extra metadata to distinguish between networks, e.g. for different internal datatypes, etc") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Scaling factors for gpu buffer size estimation @@ -347,7 +347,7 @@ AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conserv AddOption(tpcDecodingClusterRatioFactor1, float, 1.5, "", 0, "Scaling Factor: for first margin of dynamic buffer allocation for attached clusters in TPC Decoding") AddOption(tpcDecodingClusterRatioFactor2, float, 3.5, "", 0, "Scaling Factor: for second margin of dynamic buffer allocation for attached clusters in TPC Decoding") AddOption(tpcDecodingSafetyBuffer, uint16_t, 1000, "", 0, "Scaling Factor: safety cluster buffer to add to dynamic buffer allocation for attached clusters in TPC Decoding") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Debug Settings @@ -355,7 +355,7 @@ BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG" AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") AddOption(mergerDumpTrackNCl, bool, false, "", 0, "Dump file with nCl per merged track fit") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings steering the processing once the device was selected, only available on the host @@ -459,7 +459,7 @@ AddSubConfig(GPUSettingsProcessingParam, param) AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) AddSubConfig(GPUSettingsProcessingDebug, debug) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() #endif // !__OPENCL__ && !__METAL__ @@ -498,7 +498,7 @@ AddOption(perfFigure, std::string, "", "", 0, "Show as performance figure, provi AddOption(plotsDir, std::string, "plots", "", 0, "Directory to write plots to") AddOption(plotsNoTitle, bool, false, "", 0, "Do not print titles on figures") AddShortcut("compare", 0, "--QAinput", "Compare QA histograms", "-c", "--qa", "--QAinputHistogramsOnly") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings for the O2 workflow diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListDisplay.h b/GPU/GPUTracking/Definitions/GPUSettingsListDisplay.h index 587d49a712b1d..5e315360fc7d3 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListDisplay.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListDisplay.h @@ -59,7 +59,7 @@ AddOption(bFieldStepSize, float, 5.0f, "", 0, "Set field line step size") AddOption(bFieldStepCount, int32_t, 100, "", 0, "Set field line step count") AddOption(bFieldLinesCount, int32_t, 2000, "", 0, "Set field lines count") AddOption(invertColors, bool, false, "", 0, "Invert colors") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Heavy settings concerning the event display (can be changed only with rebuilding vertices) @@ -83,7 +83,7 @@ AddOption(hideUnmatchedClusters, int32_t, 0, "", 0, "Hide unmatched clusters") AddOption(trackFilter, int32_t, 0, "", 0, "Apply filter on tracks to be displayed") AddOption(projectXY, int32_t, 0, "", 0, "Project everything on the XY-plane") AddOption(xzAdd, float, 0, "", 0, "Separate sectors, increase X coordinate", min(0.f), max(300.f)) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Camera, window, and renderer settings for the event display @@ -101,7 +101,7 @@ AddOption(fpsRate, int32_t, 100, "", 0, "Target frame rate (frames per second) f AddOption(useGLIndirectDraw, bool, true, "", 0, "Use OpenGL indirect draws to reduce number of draw calls") AddOption(screenshotScaleFactor, int32_t, 1, "", 0, "Resolution scale factor when taking screenshots", min(1), max(4)) AddOption(fov, float, 45.f, "", 0, "Display FOV", min(5.f), max(175.f)) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Vulkan Display Settings @@ -109,7 +109,7 @@ BeginSubConfig(GPUSettingsDisplayVulkan, vulkan, configStandalone.display, "GLV" AddOption(nFramesInFlight, uint32_t, 3, "", 0, "Max number of render frames in flight") AddOption(uniformBuffersInDeviceMemory, bool, 1, "", 0, "Have uniform buffers in host-accessible device memory") AddOption(forceDevice, int32_t, -1, "", 0, "Force Vulkan device number to use") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Internal display settings (GUI protocol, console, info texts) @@ -123,7 +123,7 @@ AddOption(consoleHistorySize, uint32_t, 1000000, "", 0, "Number of console lines AddOption(setInfoTime, float, 5.0f, "", 0, "Time in seconds an info message (SetInfo) is shown fully opaque") AddOption(setInfoTransition, float, 1.0f, "", 0, "Time in seconds an info message fades out after setInfoTime") AddOption(consolePrintToTerminal, bool, false, "", 0, "Also write the replies of console commands to the terminal") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings concerning the event display (fixed settings, cannot be changed) @@ -143,7 +143,7 @@ AddSubConfig(GPUSettingsDisplayHeavy, heavy) AddSubConfig(GPUSettingsDisplayRenderer, renderer) AddSubConfig(GPUSettingsDisplayVulkan, vulkan) AddSubConfig(GPUSettingsDisplayInternal, internal) -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() EndNamespace() // o2::gpu diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h index 20018a2792cc0..61e2e91a55a63 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h @@ -34,13 +34,13 @@ AddOption(noEventRepeat, int32_t, 0, "", 0, "0: Place random events, 1: Place ev AddOption(nTotalEventsInTF, int32_t, 0, "", 0, "Total number of collisions to be placed in the interior of all time frames (excluding borders)") AddOption(eventStride, int32_t, 0, "", 0, "Do not select random event, but walk over array of events in stride steps") AddOption(overlayRaw, bool, false, "", 0, "Overlay raw TPC data instead of spatial clusters") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings concerning standalone toy event generator BeginSubConfig(GPUSettingsEG, EG, configStandalone, "EG", 0, "Event generator settings") AddOption(numberOfTracks, int32_t, 1, "", 0, "Number of tracks per generated event") -AddHelp("help", 'h') +AddHelp("help", 0) EndConfig() // Settings for the standalone benchmark @@ -60,7 +60,7 @@ AddOption(runsInit, int32_t, 1, "", 0, "Number of initial iterations excluded fr AddOption(eventsDir, const char*, "pp", "events", 'e', "Directory with events to process", message("Reading events from Directory %s")) AddOption(absoluteEventsDir, bool, false, "", 0, "Events directory is absolute, and not inside './events/'") AddOption(noEvents, bool, false, "", 0, "Run without data (e.g. for field visualization)") -AddOption(eventDisplay, int32_t, 0, "display", 'd', "Show standalone event display", def(1)) +AddOption(eventDisplay, int32_t, 0, "display", 0, "Show standalone event display", def(1)) AddOption(eventGenerator, bool, false, "", 0, "Run event generator") AddOption(cont, bool, false, "", 0, "Process continuous timeframe data, even if input is triggered") AddOption(setMaxTimeBin, int32_t, -2, "", 0, "maximum time bin of continuous data, 0 for triggered events, -1 for automatic continuous mode, -2 for automatic continuous / triggered") diff --git a/GPU/GPUTracking/utils/qconfig.cxx b/GPU/GPUTracking/utils/qconfig.cxx index 63212e33085ac..74fecab4371be 100644 --- a/GPU/GPUTracking/utils/qconfig.cxx +++ b/GPU/GPUTracking/utils/qconfig.cxx @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include #define QCONFIG_INTERNAL #include "qconfig.h" #undef QCONFIG_INTERNAL @@ -115,10 +117,26 @@ inline int32_t qConfigTypeSpecialized>::qAddOptionMain(qConfigS return (qAddOptionMainTupleStruct>::qAddOptionMainTuple(settings, ref, i, argv, argc)); } +template +struct qConfigArgType { // First argument of an option of this type can be omitted + static constexpr bool isTuple = false, optional = std::is_same_v; +}; +template +struct qConfigArgType> { + static constexpr bool isTuple = true, optional = std::is_same_v; +}; + template struct qConfigType : public qConfigSettingsParser { using qConfigSettingsParser::qConfigGetSettings; + template + static inline int8_t qNeedsArg(const char* /*help*/, Args&&... args) // 0 = no argument, 1 = argument required, 2 = argument optional + { + auto settings = qConfigGetSettings(args...); + return settings.doSet ? 0 : ((qConfigArgType::optional || (settings.doDefault && !qConfigArgType::isTuple)) ? 2 : 1); + } + template static inline int32_t qAddOption(T& ref, int32_t& i, const char** argv, const int argc, T def, const char* /*help*/, Args&&... args) { @@ -159,12 +177,12 @@ struct qConfigType : public qConfigSettingsParser { if (retVal) { return (retVal != qcrArgMissing || i == iFirst ? retVal : 0); } - if (i - iFirst >= count) { + if (i - iFirst > count) { printf("Too many values provided for option %s\n", getOptName(argv, i)); return (qcrArrayOverflow); } if (i == iFirst || i != iLast) { - ref[i - iFirst] = tmp; + ref[i == iFirst ? 0 : i - iFirst - 1] = tmp; // i points to the argument just read } } while (i != iLast); return (0); @@ -407,19 +425,92 @@ static inline void qConfigHelp(const char* subConfig = nullptr, int32_t followSu } // Create parser for configuration +struct qConfigOption { + int32_t handler = -1; // Index in mHandlers, -1 = no option + int8_t needsArg = 0; // 0 = no argument, 1 = argument required, 2 = argument optional + const char* forward = nullptr; // Shortcut: option to continue with after the handler +}; + +struct qConfigOptionRegistry { + std::vector> mHandlers; + std::unordered_map mLong; // Full name after -- + std::vector mShort; // Index (preoptshort << 8) | optnameshort + + void add(const char* preopt, char preoptshort, const char* name, const char* optname, char optnameshort, int8_t needsArg, const char* forward, std::function handler) + { + const qConfigOption opt{(int32_t)mHandlers.size(), needsArg, forward}; + mHandlers.emplace_back(std::move(handler)); + if (optname[0] && !mLong.try_emplace(optname, opt).second) { + throw std::runtime_error(std::string("qConfig: option name registered twice: --") + optname); + } + if (name[0] && std::string(preopt) + name != optname && !mLong.try_emplace(std::string(preopt) + name, opt).second) { + throw std::runtime_error(std::string("qConfig: option name registered twice: --") + preopt + name); + } + if (optnameshort) { + const size_t index = ((size_t)(uint8_t)preoptshort << 8) | (uint8_t)optnameshort; + if (index >= mShort.size()) { + mShort.resize(index + 1); + } + if (mShort[index].handler >= 0) { + throw std::runtime_error(std::string("qConfig: short option registered twice: -") + (preoptshort ? std::string(1, preoptshort) : std::string()) + optnameshort); + } + mShort[index] = opt; + } + } + + const qConfigOption* find(const char* option) const // --name, -x or -px + { + if (option[0] != '-' || option[1] == 0) { + return nullptr; + } + if (option[1] == '-') { + const auto it = mLong.find(&option[2]); + return it == mLong.end() ? nullptr : &it->second; + } + if (option[2] != 0 && option[3] != 0) { + return nullptr; + } + const size_t index = option[2] ? (((size_t)(uint8_t)option[1] << 8) | (uint8_t)option[2]) : (uint8_t)option[1]; + return index < mShort.size() && mShort[index].handler >= 0 ? &mShort[index] : nullptr; + } + + qConfigOptionRegistry() + { +#define QCONFIG_PARSE +#include "qconfig_internal.h" +#undef QCONFIG_PARSE + } +}; + int32_t qConfigParse(int argc, const char** argv, const char* /*filename*/) { + static const qConfigOptionRegistry registry; for (int32_t i = 1; i < argc; i++) { const char* thisoption = argv[i]; - repeat: - bool found = false; -#define QCONFIG_PARSE -#include "qconfig_internal.h" -#undef QCONFIG_PARSE - if (found == false) { - printf("Invalid argument: %s\n", argv[i]); - return (1); - goto repeat; // Suppress GCC warnings, this is never executed, label might be accessed from qconfig.h + while (true) { + const qConfigOption* opt = registry.find(thisoption); + if (opt == nullptr) { + printf("Invalid argument: %s\n", argv[i]); + return (1); + } + if (opt->forward == nullptr) { + const char* next = i + 1 < argc ? argv[i + 1] : nullptr; + if (opt->needsArg == 0 && next && next[0] != '-') { + printf("Option %s does not take an argument (%s)\n", argv[i], next); + return (qcrError); + } + if (opt->needsArg == 1 && (next == nullptr || next[0] == 0 || (next[0] == '-' && next[1] == '-'))) { + printf("Argument missing for option %s!\n", argv[i]); + return (qcrArgMissing); + } + } + if (int32_t retVal = registry.mHandlers[opt->handler](i, argv, argc)) { + return (retVal); + } + if (opt->forward == nullptr) { + break; + } + thisoption = opt->forward; } } return (0); diff --git a/GPU/GPUTracking/utils/qconfig_internal.h b/GPU/GPUTracking/utils/qconfig_internal.h index 057c45db11bdd..26076699ffda7 100644 --- a/GPU/GPUTracking/utils/qconfig_internal.h +++ b/GPU/GPUTracking/utils/qconfig_internal.h @@ -18,117 +18,62 @@ #include "qconfig_general.h" #include "qconfig_helpers.h" -#ifdef QCONFIG_PARSE +#ifdef QCONFIG_PARSE // Included in the constructor of qConfigOptionRegistry: registers the names and the handler of each option -#define QCONFIG_COMPARE(name, optname, optnameshort) \ - (thisoption[0] == '-' && ((thisoption[1] == '-' && optname[0] != 0 && strcmp(&thisoption[2], optname) == 0) || \ - (preoptshort == 0 && thisoption[1] == optnameshort && thisoption[2] == 0) || (thisoption[1] == '-' && thisoption[2] != 0 && strlen(preopt) == 0 && strcmp(&thisoption[2], name) == 0) || \ - (preoptshort != 0 && thisoption[1] == preoptshort && thisoption[2] == optnameshort && thisoption[3] == 0) || (thisoption[1] == '-' && strlen(preopt) && strncmp(&thisoption[2], preopt, strlen(preopt)) == 0 && strcmp(&thisoption[2 + strlen(preopt)], name) == 0))) +#define AddOption(name, type, default, optname, optnameshort, ...) \ + add(preopt, preoptshort, #name, optname, optnameshort, qConfigType::qNeedsArg(__VA_ARGS__), nullptr, [ref = &tmp.name](int32_t& i, const char** argv, const int argc) -> int32_t { return qConfigType::qAddOption(*ref, i, argv, argc, default, __VA_ARGS__); }); -#define AddOption(name, type, default, optname, optnameshort, ...) \ - else if (QCONFIG_COMPARE(#name, optname, optnameshort)) \ - { \ - int32_t retVal = qConfigType::qAddOption(tmp.name, i, argv, argc, default, __VA_ARGS__); \ - if (retVal) { \ - return (retVal); \ - } \ - } +#define AddOptionSet(name, type, value, optname, optnameshort, ...) \ + add(preopt, preoptshort, optname, "", optnameshort, 0, nullptr, [ref = &tmp.name](int32_t& i, const char**, const int) -> int32_t { return qConfigType::qAddOption(*ref, i, nullptr, 0, value, __VA_ARGS__, set(value)); }); -#define AddOptionSet(name, type, value, optname, optnameshort, ...) \ - else if (QCONFIG_COMPARE(optname, "", optnameshort)) \ - { \ - int32_t retVal = qConfigType::qAddOption(tmp.name, i, nullptr, 0, value, __VA_ARGS__, set(value)); \ - if (retVal) { \ - return (retVal); \ - } \ - } +#define AddOptionVec(name, type, optname, optnameshort, ...) \ + add(preopt, preoptshort, #name, optname, optnameshort, qConfigType::qNeedsArg(__VA_ARGS__), nullptr, [ref = &tmp.name](int32_t& i, const char** argv, const int argc) -> int32_t { return qConfigType::qAddOptionVec(*ref, i, argv, argc, __VA_ARGS__); }); -#define AddOptionVec(name, type, optname, optnameshort, ...) \ - else if (QCONFIG_COMPARE(#name, optname, optnameshort)) \ - { \ - int32_t retVal = qConfigType::qAddOptionVec(tmp.name, i, argv, argc, __VA_ARGS__); \ - if (retVal) { \ - return (retVal); \ - } \ - } - -#define AddOptionArray(name, type, count, default, optname, optnameshort, ...) \ - else if (QCONFIG_COMPARE(#name, optname, optnameshort)) \ - { \ - int32_t retVal = qConfigType::qAddOptionArray(tmp.name, count, i, argv, argc, __VA_ARGS__); \ - if (retVal) { \ - return (retVal); \ - } \ - } +#define AddOptionArray(name, type, count, default, optname, optnameshort, ...) \ + add(preopt, preoptshort, #name, optname, optnameshort, qConfigType::qNeedsArg(__VA_ARGS__), nullptr, [ref = &tmp.name](int32_t& i, const char** argv, const int argc) -> int32_t { return qConfigType::qAddOptionArray(*ref, count, i, argv, argc, __VA_ARGS__); }); #define AddSubConfig(name, instance) -#define BeginConfig(name, instance) \ - { \ - constexpr const char* preopt = ""; \ - constexpr const char preoptshort = 0; \ - name& tmp = instance; \ - bool tmpfound = true; \ - if (found) { \ - } +#define BeginConfig(name, instance) \ + { \ + [[maybe_unused]] constexpr const char* preopt = ""; \ + [[maybe_unused]] constexpr const char preoptshort = 0; \ + [[maybe_unused]] name& tmp = instance; #define BeginSubConfig(name, instance, parent, preoptname, preoptnameshort, descr, ...) \ { \ [[maybe_unused]] constexpr const char* preopt = preoptname; \ [[maybe_unused]] constexpr const char preoptshort = preoptnameshort; \ - [[maybe_unused]] name& tmp = parent.instance; \ - bool tmpfound = true; \ - if (found) { \ - } + [[maybe_unused]] name& tmp = parent.instance; -#define BeginHiddenConfig(name, instance) \ - { \ - bool tmpfound; \ - if (0) { \ - } +#define BeginHiddenConfig(name, instance) { -// clang-format off -#define EndConfig() \ - else { tmpfound = false; } \ - if (tmpfound) { \ - found = true; \ - } \ - } -// clang-format on +#define EndConfig() } -#define AddHelp(cmd, cmdshort) \ - else if (QCONFIG_COMPARE(cmd, "", cmdshort)) \ - { \ - const char* arg = getArg(i, argv, argc); \ - qConfigHelp(arg ? arg : preopt); \ - return (qcrHelp); \ - } +#define AddHelp(cmd, cmdshort) \ + add(preopt, preoptshort, cmd, "", cmdshort, 2, nullptr, [helpConfig = preopt](int32_t& i, const char** argv, const int argc) -> int32_t { \ + const char* arg = getArg(i, argv, argc); \ + qConfigHelp(arg ? arg : helpConfig); \ + return (qcrHelp); \ + }); -#define AddHelpAll(cmd, cmdshort) \ - else if (QCONFIG_COMPARE(cmd, "", cmdshort)) \ - { \ - const char* arg = getArg(i, argv, argc); \ - qConfigHelp(arg ? arg : "", true); \ - return (qcrHelp); \ - } +#define AddHelpAll(cmd, cmdshort) \ + add(preopt, preoptshort, cmd, "", cmdshort, 2, nullptr, [](int32_t& i, const char** argv, const int argc) -> int32_t { \ + const char* arg = getArg(i, argv, argc); \ + qConfigHelp(arg ? arg : "", true); \ + return (qcrHelp); \ + }); #define AddCommand(cmd, cmdshort, command, help) \ - else if (QCONFIG_COMPARE(cmd, "", cmdshort)) \ - { \ - if (command) { \ - return (qcrCmd); \ - } \ - } - -#define AddShortcut(cmd, cmdshort, forward, help, ...) \ - else if (QCONFIG_COMPARE(cmd, "", cmdshort)) \ - { \ - const char* options[] = {"", __VA_ARGS__, nullptr}; \ - const int32_t nOptions = sizeof(options) / sizeof(options[0]) - 1; \ - qConfigParse(nOptions, options, nullptr); \ - thisoption = forward; \ - goto repeat; \ - } + add(preopt, preoptshort, cmd, "", cmdshort, 0, nullptr, [](int32_t&, const char**, const int) -> int32_t { return (command) ? qcrCmd : 0; }); + +#define AddShortcut(cmd, cmdshort, forward, help, ...) /* Parses the listed options, then the parser continues with the forward option, which takes the arguments */ \ + add(preopt, preoptshort, cmd, "", cmdshort, 0, forward, [](int32_t&, const char**, const int) -> int32_t { \ + const char* options[] = {"", __VA_ARGS__, nullptr}; \ + const int32_t nOptions = sizeof(options) / sizeof(options[0]) - 1; \ + qConfigParse(nOptions, options, nullptr); \ + return 0; \ + }); // End QCONFIG_PARSE #elif defined(QCONFIG_HELP) From 48ce520fc5f3488cc7b98ffc7700ee7e6acd854d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 20 Nov 2026 13:35:36 +0100 Subject: [PATCH 60/60] GPU TPC: Implement TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY=1 mode, to use only the actually needed rebuildRowData rows --- .../DataTypes/GPUMemorySizeScalers.cxx | 1 + .../DataTypes/GPUMemorySizeScalers.h | 1 + GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- .../Global/GPUChainTrackingMerger.cxx | 14 ++- GPU/GPUTracking/Global/GPUErrorCodes.h | 39 +++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 14 ++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 18 ++- .../Merger/GPUTPCGMMergerRebuildInternal.h | 106 +++++++++++++++++- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 7 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 69 +++++++----- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 10 +- 11 files changed, 210 insertions(+), 72 deletions(-) diff --git a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx index 024fd9bbeb6fe..0b97c84218d8c 100644 --- a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx +++ b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.cxx @@ -80,3 +80,4 @@ size_t GPUMemorySizeScalers::NTPCSectorTrackHits(size_t tpcHits, uint8_t withRej size_t GPUMemorySizeScalers::NTPCMergedTracks(size_t tpcSectorTracks) { return getValue(tpcMaxMergedTracks, tpcSectorTracks * (conservativeMemoryEstimate ? 1.0 : tpcMergedTrackPerSectorTrack)); } size_t GPUMemorySizeScalers::NTPCMergedTrackHits(size_t tpcSectorTrackHitss) { return getValue(tpcMaxMergedTrackHits, tpcSectorTrackHitss * tpcMergedTrackHitPerSectorHit); } size_t GPUMemorySizeScalers::NTPCUnattachedHitsBase1024(int32_t type) { return (returnMaxVal || conservativeMemoryEstimate) ? 1024 : std::min(1024, tpcCompressedUnattachedHitsBase1024[type] * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor); } +size_t GPUMemorySizeScalers::NTPCRebuildRows() { return (returnMaxVal || conservativeMemoryEstimate) ? GPUTPCGeometry::NROWS : std::min(GPUTPCGeometry::NROWS, GPUTPCGeometry::NROWS * tpcTrackRebuildRowFraction * (doFuzzing == 0 ? scalingFactor : getScalingFactor()) * temporaryFactor); } diff --git a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h index a81d296459acd..2cfc6cb924ac0 100644 --- a/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h +++ b/GPU/GPUTracking/DataTypes/GPUMemorySizeScalers.h @@ -66,6 +66,7 @@ struct GPUMemorySizeScalers : public GPUSettingsProcessingScaling { size_t NTPCMergedTracks(size_t tpcSectorTracks); size_t NTPCMergedTrackHits(size_t tpcSectorTrackHitss); size_t NTPCUnattachedHitsBase1024(int32_t type); + size_t NTPCRebuildRows(); }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 91d0856718933..48074826c9245 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -343,10 +343,11 @@ AddOption(tpcSectorTrackHitsPerHitWithRejection, float, 1.0, "", 0, "Scaling Fac AddOption(tpcMergedTrackPerSectorTrack, float, 1.0, "", 0, "Scaling Factor: tpcMergedTrackPerSectorTrack") AddOption(tpcMergedTrackHitPerSectorHit, float, 1.1, "", 0, "Scaling Factor: tpcMergedTrackHitPerSectorHit") AddOptionArray(tpcCompressedUnattachedHitsBase1024, int32_t, 3, (900, 900, 500), "", 0, "Scaling Factor: tpcCompressedUnattachedHitsBase1024") -AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conservative defaults for larger buffers during TPC processing") AddOption(tpcDecodingClusterRatioFactor1, float, 1.5, "", 0, "Scaling Factor: for first margin of dynamic buffer allocation for attached clusters in TPC Decoding") AddOption(tpcDecodingClusterRatioFactor2, float, 3.5, "", 0, "Scaling Factor: for second margin of dynamic buffer allocation for attached clusters in TPC Decoding") +AddOption(tpcTrackRebuildRowFraction, float, 0.69, "", 0, "Scaling Factor: Fraction of rows for which to allocate rebuild memory") // 87 of 152 in average with 20% margin AddOption(tpcDecodingSafetyBuffer, uint16_t, 1000, "", 0, "Scaling Factor: safety cluster buffer to add to dynamic buffer allocation for attached clusters in TPC Decoding") +AddOption(conservativeMemoryEstimate, bool, false, "", 0, "Use some more conservative defaults for larger buffers during TPC processing") AddHelp("help", 0) EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 634520e2f90a1..32cad4d62b82c 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -207,11 +207,11 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } - const bool rebuildConstrainedRows = param().rec.tpc.rebuildTrackInFit && mRec->getGPUParameters(doGPU).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY; if (param().rec.tpc.rebuildTrackInFit) { mRec->PushNonPersistentMemory(qStr2Tag("TPCREBLD")); // The rebuild memory is freed after the fit, before the O2 output } - if (rebuildConstrainedRows) { + const int32_t rebuildConstrainedRows = param().rec.tpc.rebuildTrackInFit ? mRec->getGPUParameters(doGPU).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY : 0; // 1: row ranges estimated before the fit, 2: filled row ranges stored by the fit (host only) + if (rebuildConstrainedRows == 1) { AllocateRegisteredMemory(Merger.MemoryResRebuild2()); WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); } @@ -222,7 +222,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel(GetGridAuto(0, deviceType)); runKernel(GetGridAuto(0, deviceType)); runKernel(GetGridAuto(0, deviceType)); - if (doGPU && rebuildConstrainedRows) { // PrepareForFit1 computed nRebuildTrackRow, needed on the host to allocate the rebuild memory, waited for below + if (doGPU && rebuildConstrainedRows == 1) { // PrepareForFit1 computed nRebuildTrackRow, needed on the host to allocate the rebuild memory, waited for below ReleaseEvent(mEvents->single); TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0, &mEvents->single); waitForTransfer = 1; @@ -249,9 +249,11 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (param().rec.tpc.rebuildTrackInFit) { AllocateRegisteredMemory(Merger.MemoryResRebuild()); // With constrained rows sized by nRebuildTrackRow, which was transferred to the host above WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); - const size_t nRebuildRows = rebuildConstrainedRows ? (size_t)Merger.Memory()->nRebuildTrackRow : (size_t)Merger.NMergedTracks() * GPUTPCGeometry::NROWS; - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), nRebuildRows * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), nRebuildRows * sizeof(*MergerShadow.TrackRebuildHelperRow())); + if (rebuildConstrainedRows != 2) { // 2: no cleaning, the fit stores complete rows and untouched memory shall not become physical + const size_t nRebuildRows = rebuildConstrainedRows == 1 ? (size_t)Merger.Memory()->nRebuildTrackRow : (size_t)Merger.NMergedTracks() * GPUTPCGeometry::NROWS; + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), nRebuildRows * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), nRebuildRows * sizeof(*MergerShadow.TrackRebuildHelperRow())); + } } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index b3894cf267c17..94f8ff7ab0cdc 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -34,24 +34,25 @@ GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(26, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(26, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER -GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped -GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded -GPUCA_ERROR_CODE(29, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete -GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset -GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word -GPUCA_ERROR_CODE(32, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode -GPUCA_ERROR_CODE(33, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version -GPUCA_ERROR_CODE(34, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder -GPUCA_ERROR_CODE(34, MAX_GPUCA_ERROR_NUMBER) +GPUCA_ERROR_CODE(18, ERROR_MERGER_REBUILD_ROW_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(19, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(27, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(27, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER +GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped +GPUCA_ERROR_CODE(29, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded +GPUCA_ERROR_CODE(30, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete +GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset +GPUCA_ERROR_CODE(32, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word +GPUCA_ERROR_CODE(33, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode +GPUCA_ERROR_CODE(34, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version +GPUCA_ERROR_CODE(35, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder +GPUCA_ERROR_CODE(35, MAX_GPUCA_ERROR_NUMBER) // #define GPUCA_CHECK_TPCZS_CORRUPTION diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 6393aa82c28fe..103cc4998e97c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -428,20 +428,26 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) void* GPUTPCGMMerger::SetPointersRebuild(void* mem) { - const bool constrained = mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY; - const uint64_t nRows = constrained ? (uint64_t)HostProcessor(this).mMemory->nRebuildTrackRow : (uint64_t)HostProcessor(this).NMergedTracks() * GPUTPCGeometry::NROWS; + const int32_t mode = mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY; // 1: sum of the estimated row ranges, 2: scaled, the fit stores the filled row ranges + const uint64_t nRows = mode == 1 ? (uint64_t)HostProcessor(this).mMemory->nRebuildTrackRow : (uint64_t)HostProcessor(this).NMergedTracks() * (mode == 2 ? mRec->MemoryScalers()->NTPCRebuildRows() : GPUTPCGeometry::NROWS); if (nRows * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates > 0xFFFFFFFFu) { throw std::bad_alloc(); } computePointerWithAlignment(mem, mClusterCandidates, nRows * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); computePointerWithAlignment(mem, mTrackRebuildHelperRow, nRows); + mNRebuildTrackRow = nRows; computePointerWithAlignment(mem, mTrackRebuildHelper, HostProcessor(this).NMergedTracks()); + if (mode == 2) { + computePointerWithAlignment(mem, mTrackRebuildHelperRowInfo, HostProcessor(this).NMergedTracks()); + } return mem; } void* GPUTPCGMMerger::SetPointersRebuild2(void* mem) { - computePointerWithAlignment(mem, mTrackRebuildHelperRowInfo, mNMaxTracks); // Needed before the row ranges are known, only with par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY + if (mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 1) { + computePointerWithAlignment(mem, mTrackRebuildHelperRowInfo, mNMaxTracks); // Needed before the row ranges are known + } return mem; } @@ -1910,7 +1916,7 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in } CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } - if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 1) { if (Param().rec.tpc.rebuildTrackInFit) { // Row range the rebuild can use: cluster rows and the radii the helix can reach, plus margin const auto& GPUrestrict() p = trk.GetParam(); const float k = -p.GetQPt() * Param().bzCLight; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index d483f34984fc9..f35e231d6dbc7 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -65,7 +65,7 @@ class GPUTPCGMMerger : public GPUProcessor ~GPUTPCGMMerger() = default; GPUTPCGMMerger(const GPUTPCGMMerger&) = delete; const GPUTPCGMMerger& operator=(const GPUTPCGMMerger&) const = delete; - static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors + static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; struct memory { GPUAtomic(uint32_t) nRetryRefit; @@ -73,7 +73,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; GPUAtomic(uint32_t) nMergedTrackClusters; - GPUAtomic(uint32_t) nRebuildTrackRow; // Sum of the row ranges of all tracks for the rebuild + GPUAtomic(uint32_t) nRebuildTrackRow; GPUAtomic(uint32_t) nO2Tracks; GPUAtomic(uint32_t) nO2ClusRefs; const GPUTPCTrack* firstExtrapolatedTracks[NSECTORS]; @@ -87,7 +87,7 @@ class GPUTPCGMMerger : public GPUProcessor uint8_t sector; }; - struct trackRebuildHelperCandidate; // Rebuild structs are defined in GPUTPCGMMergerRebuildInternal.h + struct trackRebuildHelperCandidate; struct trackRebuildHelper; struct trackRebuildHelperRowInfo; struct trackRebuildHelperRow; @@ -141,6 +141,17 @@ class GPUTPCGMMerger : public GPUProcessor GPUdi() trackRebuildHelperCandidate* ClusterCandidates(int32_t iTrk, uint32_t row) const; GPUdi() trackRebuildHelperRow& TrackRebuildHelperRow(int32_t iTrk, uint32_t row) const; GPUdi() bool RebuildRowInRange(int32_t iTrk, uint32_t row) const; + // Variants for the fit: use the temporary rows of the track if tmpRows is set (only with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) + template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty + GPUdi() trackRebuildHelperCandidate* ClusterCandidates(int32_t iTrk, uint32_t row, T* tmpRows) const; + template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty + GPUdi() trackRebuildHelperRow& TrackRebuildHelperRow(int32_t iTrk, uint32_t row, T* tmpRows) const; + template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty + GPUdi() bool RebuildRowInRange(int32_t iTrk, uint32_t row, T* tmpRows) const; + template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty + GPUdi() void ClearRebuildTmpRows(int32_t iTrk, T* tmpRows) const; + template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty + GPUdi() bool StoreRebuildTmpRows(int32_t iTrk, const T* tmpRows); GPUdi() uint32_t RebuildFirstRow(int32_t iTrk) const; // Row range of the track for the rebuild, all rows without GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY GPUdi() uint32_t RebuildLastRow(int32_t iTrk) const; GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } @@ -292,6 +303,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t mNTotalSectorTracks = 0; // maximum number of incoming sector tracks uint32_t mNMaxTracks = 0; // maximum number of output tracks + uint32_t mNRebuildTrackRow = 0; // number of rows allocated for the rebuild uint32_t mNMaxSingleSectorTracks = 0; // max N tracks in one sector uint32_t mNMaxMergedTrackClusters = 0; // max number of clusters in output tracks (double-counting shared clusters) uint32_t mNClusters = 0; // max total unique clusters (in event) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h index 1d4cc8de0c454..6ff2870dbb92c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h @@ -17,6 +17,7 @@ #include "GPUTPCGMMerger.h" #include "GPUCommonMath.h" +#include "GPUErrors.h" namespace o2::gpu { @@ -43,9 +44,9 @@ struct GPUTPCGMMerger::trackRebuildHelperCandidate { // Per track, row and candi }; static_assert(sizeof(GPUTPCGMMerger::trackRebuildHelperCandidate) == 6); -struct __attribute__((packed)) GPUTPCGMMerger::trackRebuildHelperRowInfo { // Per track, only with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY +struct __attribute__((packed)) GPUTPCGMMerger::trackRebuildHelperRowInfo { // Per track, only with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1 uint32_t firstRowIndex; // Index of firstRow of the track in mTrackRebuildHelperRow and (times the number of candidates) in mClusterCandidates - uint8_t firstRow, lastRow; // Row range the rebuild can use for the track, set in PrepareForFit1 + uint8_t firstRow, lastRow; // Row range the rebuild can use for the track: 1 = estimate from PrepareForFit1, 2 = range filled by the candidate search of the fit, empty (firstRow > lastRow) otherwise }; struct GPUTPCGMMerger::trackRebuildHelper { // Per track @@ -64,6 +65,15 @@ struct GPUTPCGMMerger::trackRebuildHelperRow { // Per track and row: weight (low GPUdi() int32_t setWeight(float w) { return (data = (data & 0xF0000000) | (w > 0.5f ? (w < 2147483648.f ? CAMath::Float2UIntReint(w) - 0x3F000000 : 0x0FFFFFFF) : 0)) & 0x0FFFFFFF; } }; +template +struct GPUTPCGMMergerRebuildTmpRows { // Empty unless tmpMode == 2, used as GPUTPCGMMergerRebuildTmp +}; +template <> +struct GPUTPCGMMergerRebuildTmpRows<2> { // Rows of one track during the candidate search of the fit with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2 (host only), the filled row range is stored to the merger after the fit + GPUTPCGMMerger::trackRebuildHelperRow rows[GPUTPCGeometry::NROWS]; + GPUTPCGMMerger::trackRebuildHelperCandidate candidates[GPUTPCGeometry::NROWS * (GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved - 1)]; // For the maximum number of candidates, only the part for rebuildTrackInFitClusterCandidates is used +}; + GPUdi() uint32_t GPUTPCGMMerger::RebuildFirstRow(int32_t iTrk) const { if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { @@ -85,7 +95,7 @@ GPUdi() uint32_t GPUTPCGMMerger::RebuildLastRow(int32_t iTrk) const GPUdi() bool GPUTPCGMMerger::RebuildRowInRange(int32_t iTrk, uint32_t row) const { if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY >= 1) { - return row - mTrackRebuildHelperRowInfo[iTrk].firstRow <= (uint32_t)(mTrackRebuildHelperRowInfo[iTrk].lastRow - mTrackRebuildHelperRowInfo[iTrk].firstRow); + return row >= mTrackRebuildHelperRowInfo[iTrk].firstRow && row <= mTrackRebuildHelperRowInfo[iTrk].lastRow; // The range can be empty (firstRow > lastRow) } else { return true; // No guard needed, the memory covers all rows } @@ -109,6 +119,96 @@ GPUdi() GPUTPCGMMerger::trackRebuildHelperRow& GPUTPCGMMerger::TrackRebuildHelpe } } +template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty +GPUdi() bool GPUTPCGMMerger::RebuildRowInRange(int32_t iTrk, uint32_t row, T* tmpRows) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) { + if (tmpRows) { + return true; + } + } + return RebuildRowInRange(iTrk, row); +} + +template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty +GPUdi() GPUTPCGMMerger::trackRebuildHelperCandidate* GPUTPCGMMerger::ClusterCandidates(int32_t iTrk, uint32_t row, T* tmpRows) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) { + if (tmpRows) { + return &tmpRows->candidates[row * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + } + } + return ClusterCandidates(iTrk, row); +} + +template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty +GPUdi() GPUTPCGMMerger::trackRebuildHelperRow& GPUTPCGMMerger::TrackRebuildHelperRow(int32_t iTrk, uint32_t row, T* tmpRows) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) { + if (tmpRows) { + return tmpRows->rows[row]; + } + } + return TrackRebuildHelperRow(iTrk, row); +} + +template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty +GPUdi() void GPUTPCGMMerger::ClearRebuildTmpRows(int32_t iTrk, T* tmpRows) const +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) { + mTrackRebuildHelperRowInfo[iTrk] = trackRebuildHelperRowInfo{0, 1, 0}; // Empty row range, set by StoreRebuildTmpRows + for (uint32_t i = 0; i < GPUTPCGeometry::NROWS; i++) { + tmpRows->rows[i] = trackRebuildHelperRow{}; + } + for (uint32_t i = 0; i < GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) { + tmpRows->candidates[i] = trackRebuildHelperCandidate{}; + } + } +} + +template // T = GPUTPCGMMergerRebuildTmp, template only to discard the code if it is empty +GPUdi() bool GPUTPCGMMerger::StoreRebuildTmpRows(int32_t iTrk, const T* tmpRows) // Returns false if the rebuild memory is full. Stores the row range of the track: the rows filled by the fit and the rows of its clusters (PrepareHitWeights adds those as candidates) +{ + if constexpr (GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2) { + const uint32_t nCandidates = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + const auto& GPUrestrict() trk = mMergedTracks[iTrk]; + int32_t first = GPUTPCGeometry::NROWS, last = -1; + for (uint32_t i = 0; i < trk.NClusters(); i++) { + first = CAMath::Min(first, mClusters[trk.FirstClusterRef() + i].row); + last = CAMath::Max(last, mClusters[trk.FirstClusterRef() + i].row); + } + for (int32_t i = 0; i < first; i++) { + if (tmpRows->rows[i].data != 0 || !tmpRows->candidates[i * nCandidates].isEmpty()) { + first = i; + } + } + for (int32_t i = GPUTPCGeometry::NROWS - 1; i > last; i--) { + if (tmpRows->rows[i].data != 0 || !tmpRows->candidates[i * nCandidates].isEmpty()) { + last = i; + } + } + if (first > last) { + return true; // Range stays empty + } + const uint32_t index = CAMath::AtomicAdd(&mMemory->nRebuildTrackRow, (uint32_t)(last - first + 1)); + if (index + last - first + 1 > mNRebuildTrackRow) { + raiseError(GPUErrors::ERROR_MERGER_REBUILD_ROW_OVERFLOW, index + last - first + 1, mNRebuildTrackRow); + return false; // Range stays empty + } + auto& GPUrestrict() info = mTrackRebuildHelperRowInfo[iTrk]; + info.firstRow = first; + info.lastRow = last; + info.firstRowIndex = index; + for (int32_t i = 0; i <= last - first; i++) { + mTrackRebuildHelperRow[info.firstRowIndex + i] = tmpRows->rows[first + i]; + } + for (uint32_t i = 0; i < (last - first + 1) * nCandidates; i++) { + mClusterCandidates[info.firstRowIndex * nCandidates + i] = tmpRows->candidates[first * nCandidates + i]; + } + } + return true; +} + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 510583f01bc93..249d61bfc3b90 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -19,6 +19,13 @@ #include "GPUCommonMath.h" #include "GPUGeneralKernels.h" +namespace o2::gpu +{ +template +struct GPUTPCGMMergerRebuildTmpRows; // Defined in GPUTPCGMMergerRebuildInternal.h +using GPUTPCGMMergerRebuildTmp = GPUTPCGMMergerRebuildTmpRows; // Temporary rows of a track in the fit, empty unless the parameter is 2 +} // namespace o2::gpu + namespace o2::gpu::gputpcgmmergertypes { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 9e5cdbdcc139a..ca4eeb24d2649 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -58,7 +58,7 @@ using namespace o2::gpu; using namespace o2::tpc; template -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMMergerRebuildTmp* tmpRows, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; @@ -196,9 +196,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } - FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, tmpRows, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); if (crossCE && doInterpolate == 2) { // Track crosses the CE within the row gap, search also on the other side, the sector there has the same alpha - FitAddRow(iRow, sector >= GPUTPCGeometry::NSECTORS / 2 ? sector - GPUTPCGeometry::NSECTORS / 2 : sector + GPUTPCGeometry::NSECTORS / 2, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, false, false, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + FitAddRow(iRow, sector >= GPUTPCGeometry::NSECTORS / 2 ? sector - GPUTPCGeometry::NSECTORS / 2 : sector + GPUTPCGeometry::NSECTORS / 2, iTrk, track, prop, inFlyDirection, merger, tmpRows, &dEdxSubThresholdRow, false, false, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); } } } @@ -217,7 +217,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ NTolerated++; MarkAdjacentSearch(param, iWay, inter); if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); + FindBestInterpolatedHit(merger, tmpRows, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } continue; } @@ -227,7 +227,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); + InterpolateMissingRows(merger, tmpRows, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } MarkAdjacentSearch(param, iWay, inter); fitBroken = true; @@ -236,7 +236,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps + uncorrectedY = FindBestInterpolatedHit(merger, tmpRows, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && (retryAttempt || !inter.isValid() || (ihit - ihitStartInterpolateReject) * wayDirection > 0 || (param.rec.tpc.disableMarkAdjacent & 1))) : allowChangeClusters)) { // With interpolation rejection: in the last way all hits InterpolateReject did not search: in a retry, beyond its first hit, or without interpolated position (also invalidated where it skipped a hit) uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } @@ -298,7 +298,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissedHard, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, tmpRows, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissedHard, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { // TODO: Could be investigated to reinit the linearization also when sinPhi drifts away from it (a reinit when QPt differs makes things worse, early in the way the QPt of the linearization is usually the better one) @@ -319,7 +319,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } else if (retValHit == 1) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); + InterpolateMissingRows(merger, tmpRows, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } fitBroken = true; break; @@ -348,7 +348,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ bool revert = false; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; - for (; iExtrapolatedRow < GPUTPCGeometry::NROWS && merger.RebuildRowInRange(iTrk, iExtrapolatedRow); iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + for (; iExtrapolatedRow < GPUTPCGeometry::NROWS && merger.RebuildRowInRange(iTrk, iExtrapolatedRow, tmpRows); iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers bool fail = false; for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; @@ -392,8 +392,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); - float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); - auto& candidate = merger.ClusterCandidates(iTrk, iExtrapolatedRow)[0]; + float uncorrectedY = FindBestInterpolatedHit(merger, tmpRows, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); + auto& candidate = merger.ClusterCandidates(iTrk, iExtrapolatedRow, tmpRows)[0]; bool found = candidate.hasCluster(); if (found) { lastExtrapolateRow = iExtrapolatedRow; @@ -407,7 +407,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[merger.CandidateClusterId(candidate, iExtrapolatedRow)]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { - merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow, tmpRows).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; break; } @@ -426,12 +426,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else - merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow, tmpRows).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { - merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iExtrapolatedRow, tmpRows).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { revert = true; break; @@ -475,7 +475,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { - merger.TrackRebuildHelperRow(iTrk, iRow).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); + merger.TrackRebuildHelperRow(iTrk, iRow, tmpRows).setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); } } if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped @@ -541,7 +541,7 @@ GPUdi() void GPUTPCGMTrackParam::MarkAdjacentSearch(const GPUParam& GPUrestrict( } } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMMergerRebuildTmp* tmpRows, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; @@ -561,7 +561,7 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { inter.markInvalid(); } - FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, doAttach ? (1 + (track.Leg() == 0)) : 0); + FindBestInterpolatedHit(merger, tmpRows, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, doAttach ? (1 + (track.Leg() == 0)) : 0); } else { if (doInterpolate == 1) { prop.InterpolateFill(&inter); @@ -588,7 +588,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissedHard, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMMergerRebuildTmp* tmpRows, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissedHard, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -628,8 +628,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger rejectChi2 = true; } } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { - if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid() && merger.RebuildRowInRange(iTrk, cluster.row)) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut - auto& candidate = merger.ClusterCandidates(iTrk, cluster.row)[0]; + if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid() && merger.RebuildRowInRange(iTrk, cluster.row, tmpRows)) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates(iTrk, cluster.row, tmpRows)[0]; if (!candidate.hasCluster() && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation candidate.set(merger.CandidateId(cluster.num, cluster.sector, cluster.row), cluster.sector, (CAMath::half_t)(CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])))); @@ -750,7 +750,7 @@ GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUre return 0; } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMMergerRebuildTmp* tmpRows, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -764,7 +764,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; } const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; - if ((useInter || useCurrent) && merger.RebuildRowInRange(iTrk, row)) { // No candidates for rows outside of the row range of the track + if ((useInter || useCurrent) && merger.RebuildRowInRange(iTrk, row, tmpRows)) { // No candidates for rows outside of the row range of the track const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -813,7 +813,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } int32_t nCandidates = 0; - while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates(iTrk, row)[nCandidates].hasCluster()) { + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates(iTrk, row, tmpRows)[nCandidates].hasCluster()) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { @@ -869,26 +869,26 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { - if (err < (float)merger.ClusterCandidates(iTrk, row)[c].error) { + if (err < (float)merger.ClusterCandidates(iTrk, row, tmpRows)[c].error) { insert = c; break; } } if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger.ClusterCandidates(iTrk, row)[c] = merger.ClusterCandidates(iTrk, row)[c - 1]; + merger.ClusterCandidates(iTrk, row, tmpRows)[c] = merger.ClusterCandidates(iTrk, row, tmpRows)[c - 1]; } // TODO: Downweight non-interpolated candidates - merger.ClusterCandidates(iTrk, row)[insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); + merger.ClusterCandidates(iTrk, row, tmpRows)[insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } - CADEBUG(const auto* dbgCand = merger.ClusterCandidates(iTrk, row); for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = merger.ClusterCandidates(iTrk, row, tmpRows); for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); } if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working - merger.ClusterCandidates(iTrk, row)[0].setNoCandidate(); + merger.ClusterCandidates(iTrk, row, tmpRows)[0].setNoCandidate(); } } else if (doAttach && (!interOnly || inter.isValid())) { const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; @@ -902,7 +902,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre return uncorrectedY; } -GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, const int8_t doAttach) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMMergerRebuildTmp* tmpRows, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, const int8_t doAttach) { for (; ihit >= 0; ihit--, interpolationIndex--) { while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { @@ -917,7 +917,7 @@ GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrest return; } auto inter = interpolation.hit[interpolationIndex]; - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, doAttach); + FindBestInterpolatedHit(merger, tmpRows, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, doAttach); } } @@ -1519,7 +1519,11 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track); + GPUTPCGMMergerRebuildTmp tmpRowsData, *tmpRows = (merger.Param().rec.tpc.rebuildTrackInFit && !rebuilt) ? &tmpRowsData : nullptr; // Filled by the candidate search, used only with GPUCA_PAR_TPC_MERGER_REBUILD_CONSTRAINED_ROW_MEMORY == 2 + if (tmpRows) { + merger.ClearRebuildTmpRows(iTrk, tmpRows); + } + bool ok = t.Fit(merger, tmpRows, iTrk, nTrackHits, NTolerated, Alpha, track); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { @@ -1541,6 +1545,9 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); + if (ok && tmpRows && !merger.StoreRebuildTmpRows(iTrk, tmpRows)) { + ok = false; // Rebuild memory overflow, error raised + } track.SetOK(ok); if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 31299b2fb8a72..61a3331a79c08 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,10 +155,10 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; template - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger& merger, GPUTPCGMMergerRebuildTmp* tmpRows, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, GPUTPCGMMergerRebuildTmp* tmpRows, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, GPUTPCGMMergerRebuildTmp* tmpRows, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void MarkAdjacentSearch(const GPUParam& param, int32_t iWay, gputpcgmmergertypes::InterpolationErrorHit& inter); // For hits the interpolation rejection way skips template @@ -167,8 +167,8 @@ class GPUTPCGMTrackParam GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg), marks inter as done - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); - GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, GPUTPCGMMergerRebuildTmp* tmpRows, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, GPUTPCGMMergerRebuildTmp* tmpRows, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z);