From 7f18efec43adec05d0f562813feafff64653bf55 Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Wed, 26 Jul 2023 13:34:58 +0200 Subject: [PATCH 0001/1239] Fix data type configuration for trackQC. (#1153) Co-authored-by: Mattia Faggin --- .../json/analysis-testing-EventTrackQA-data.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json b/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json index d052e87c7..52c800a8b 100644 --- a/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json +++ b/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json @@ -92,13 +92,15 @@ "processStandard": "false" }, "track-selection": { - "compatibilityIU": "false", - "etaMax": "0.800000012", - "etaMin": "-0.800000012", - "isRun3": "true", + "compatibilityIU": "0", + "dcaSetup": "0", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "1", "itsMatching": "1", - "produceFBextendedTable": "true", + "produceFBextendedTable": "-1", + "produceTable": "1", "ptMax": "1e+10", - "ptMin": "0.100000001" + "ptMin": "0.1" } } From 9eaeef0d0576c4ecd59cab5f868073937ca233ee Mon Sep 17 00:00:00 2001 From: mcoquet642 <74600025+mcoquet642@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:47:16 +0200 Subject: [PATCH 0002/1239] Adding MCH-MID tracks to AOD (#1150) * Adding MCH-MID tracks to aod production --------- Co-authored-by: Benedikt Volkel --- MC/bin/o2dpg_sim_workflow.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 4e5b02db8..01c347281 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1098,6 +1098,9 @@ def getDigiTaskName(det): if isActive("MID"): pvfinder_matching_sources += ",MID" pvfinderneeds += [MIDRECOtask['name']] + if isActive('MCH') and isActive('MID'): + pvfinder_matching_sources += ",MCH-MID" + pvfinderneeds += [MCHMIDMATCHtask['name']] if isActive('FT0'): pvfinderneeds += [FT0RECOtask['name']] @@ -1314,6 +1317,9 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if isActive('MID'): aodneeds += [ MIDRECOtask['name'] ] aodinfosources += ',MID' + if isActive('MID') and isActive('MCH'): + aodneeds += [ MCHMIDMATCHtask['name'] ] + aodinfosources += ',MCH-MID' if args.with_ZDC and isActive('ZDC'): aodneeds += [ ZDCRECOtask['name'] ] aodinfosources += ',ZDC' From 56ec8bb02a9f09e6bb7b69e42c5c021cc59c3b6a Mon Sep 17 00:00:00 2001 From: Raphaelle Bailhache Date: Thu, 27 Jul 2023 09:57:56 +0200 Subject: [PATCH 0003/1239] Update of Gap trigger HF enhance MC --- .../Generator_pythia8_GapTriggered_HFLepton.C | 2 +- .../PWGEM/pythia8/generator/pythia8_bbbar.cfg | 36 +++++++++++++++ .../pythia8_bbbar_forceddecayscharmbeauty.cfg | 19 ++++++++ MC/run/PWGEM/runHFGapToDielectrons_pp.sh | 45 +++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 MC/config/PWGEM/pythia8/generator/pythia8_bbbar.cfg create mode 100644 MC/run/PWGEM/runHFGapToDielectrons_pp.sh diff --git a/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C b/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C index 74e6313d8..ce8dae943 100644 --- a/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C +++ b/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C @@ -191,7 +191,7 @@ FairGenerator *GeneratorPythia8GapTriggeredBeautyForcedDecays(int inputTriggerRa // Beauty-enriched no forced decay FairGenerator *GeneratorPythia8GapTriggeredBeautyNoForcedDecays(int inputTriggerRatio, float yMin=-1.5, float yMax=1.5) { - auto myGen = new GeneratorPythia8GapTriggeredHFLepton("$O2DPG_ROOT/MC/config/common/pythia8/generator/pythia8_bbbar.cfg", 5, inputTriggerRatio); + auto myGen = new GeneratorPythia8GapTriggeredHFLepton("$O2DPG_ROOT/MC/config/PWGEM/pythia8/generator/pythia8_bbbar.cfg", 5, inputTriggerRatio); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); diff --git a/MC/config/PWGEM/pythia8/generator/pythia8_bbbar.cfg b/MC/config/PWGEM/pythia8/generator/pythia8_bbbar.cfg new file mode 100644 index 000000000..52044834b --- /dev/null +++ b/MC/config/PWGEM/pythia8/generator/pythia8_bbbar.cfg @@ -0,0 +1,36 @@ +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +#HardQCD:hardccbar on # scatterings g-g / q-qbar -> c-cbar +HardQCD:hardbbbar on # scatterings g-g / q-qbar -> b-bbar + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### switch on color reconnection in mode 2 (https://arxiv.org/pdf/1505.01681.pdf) +Tune:pp = 14 +ColourReconnection:mode = 1 +ColourReconnection:allowDoubleJunRem = off +ColourReconnection:m0 = 0.3 +ColourReconnection:allowJunctions = on +ColourReconnection:junctionCorrection = 1.20 +ColourReconnection:timeDilationMode = 2 +ColourReconnection:timeDilationPar = 0.18 +StringPT:sigma = 0.335 +StringZ:aLund = 0.36 +StringZ:bLund = 0.56 +StringFlav:probQQtoQ = 0.078 +StringFlav:ProbStoUD = 0.2 +StringFlav:probQQ1toQQ0join = 0.0275,0.0275,0.0275,0.0275 +MultiPartonInteractions:pT0Ref = 2.15 +BeamRemnants:remnantMode = 1 +BeamRemnants:saturation =5 + +# Correct OmegaC decay length (wrong in PYTHIA8 decay table) (mm/c) +4332:tau0 = 0.08000000000 +# Correct Lb decay length (wrong in PYTHIA8 decay table) +5122:tau0 = 4.41000e-01 diff --git a/MC/config/PWGEM/pythia8/generator/pythia8_bbbar_forceddecayscharmbeauty.cfg b/MC/config/PWGEM/pythia8/generator/pythia8_bbbar_forceddecayscharmbeauty.cfg index 4f7baf967..ae478248b 100644 --- a/MC/config/PWGEM/pythia8/generator/pythia8_bbbar_forceddecayscharmbeauty.cfg +++ b/MC/config/PWGEM/pythia8/generator/pythia8_bbbar_forceddecayscharmbeauty.cfg @@ -11,6 +11,25 @@ HardQCD:hardbbbar on # scatterings g-g / q-qbar -> b-bbar ParticleDecays:limitTau0 on ParticleDecays:tau0Max 10. +### switch on color reconnection in mode 2 (https://arxiv.org/pdf/1505.01681.pdf) +Tune:pp = 14 +ColourReconnection:mode = 1 +ColourReconnection:allowDoubleJunRem = off +ColourReconnection:m0 = 0.3 +ColourReconnection:allowJunctions = on +ColourReconnection:junctionCorrection = 1.20 +ColourReconnection:timeDilationMode = 2 +ColourReconnection:timeDilationPar = 0.18 +StringPT:sigma = 0.335 +StringZ:aLund = 0.36 +StringZ:bLund = 0.56 +StringFlav:probQQtoQ = 0.078 +StringFlav:ProbStoUD = 0.2 +StringFlav:probQQ1toQQ0join = 0.0275,0.0275,0.0275,0.0275 +MultiPartonInteractions:pT0Ref = 2.15 +BeamRemnants:remnantMode = 1 +BeamRemnants:saturation =5 + ### only semileptonic decays for charm ### D+ 411:oneChannel = 1 0.087 0 -311 -11 12 diff --git a/MC/run/PWGEM/runHFGapToDielectrons_pp.sh b/MC/run/PWGEM/runHFGapToDielectrons_pp.sh new file mode 100644 index 000000000..b83c839e3 --- /dev/null +++ b/MC/run/PWGEM/runHFGapToDielectrons_pp.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- SETUP LOCAL CCDB CACHE -------------------------- +export ALICEO2_CCDB_LOCALCACHE=$PWD/.ccdb + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-500000} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +#generate random number +RNDSIG=$(($RANDOM % 100)) +echo $RNDSIG + +if [[ $RNDSIG -ge 0 && $RNDSIG -lt 20 ]]; +then + CONFIGNAME="GeneratorHFGapTriggered_Charm.ini" +elif [[ $RNDSIG -ge 20 && $RNDSIG -lt 40 ]]; +then + CONFIGNAME="GeneratorHFGapTriggered_BeautyForcedDecay.ini" +elif [[ $RNDSIG -ge 40 && $RNDSIG -lt 100 ]]; +then + CONFIGNAME="GeneratorHFGapTriggered_BeautyNoForcedDecay.ini" +fi + + + + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -col pp -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + ${SEED} \ + -ini $O2DPG_ROOT/MC/config/PWGEM/ini/$CONFIGNAME \ + -confKeyBkg "Diamond.width[2]=6" -interactionRate ${INTRATE} + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 From 46bb3aa918b7613aadd1aa7818dc944c6ad90098 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Mon, 24 Jul 2023 16:49:50 +0200 Subject: [PATCH 0004/1239] Filter task names in stacked plots --- MC/utils/o2dpg_sim_metrics.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/MC/utils/o2dpg_sim_metrics.py b/MC/utils/o2dpg_sim_metrics.py index 448dfaf39..6631c0bc2 100755 --- a/MC/utils/o2dpg_sim_metrics.py +++ b/MC/utils/o2dpg_sim_metrics.py @@ -558,7 +558,7 @@ def resources_per_iteration(resources, fields, task_filter=None, per_what=None): return list(range(start, end + 1)), values -def plot_resource_history(json_pipelines, out_dir, filter=None, suffix="", labels=None): +def plot_resource_history(json_pipelines, out_dir, task_filter=None, suffix="", labels=None): """ Plotting resource history @@ -595,7 +595,7 @@ def plot_resource_history(json_pipelines, out_dir, filter=None, suffix="", label name = labels[jp_i] n_cpu = jp.meta["cpu_limit"] - iterations, iterations_y = resources_per_iteration(jp, metrics, filter) + iterations, iterations_y = resources_per_iteration(jp, metrics, task_filter) names.append(f"{jp_i}_{name}") @@ -629,7 +629,7 @@ def plot_resource_history(json_pipelines, out_dir, filter=None, suffix="", label save_figure(figure, join(out_dir, f"{me}_min_max_average{suffix}.png")) -def plot_resource_history_stacked(res, out_dir, per_what): +def plot_resource_history_stacked(res, out_dir, per_what, task_filter=None): """ Plotting resource history @@ -650,7 +650,7 @@ def plot_resource_history_stacked(res, out_dir, per_what): axes.append(ax) n_cpu = res.meta["cpu_limit"] - iterations, iterations_y = resources_per_iteration(res, metrics, per_what=per_what) + iterations, iterations_y = resources_per_iteration(res, metrics, task_filter, per_what=per_what) # only print every modulo iteration on the x-axis modulo = 10**(max(0, len(str(len(iterations))) - 2)) @@ -777,11 +777,11 @@ def history(args): # make stacked bar charts over iterations # per task - plot_resource_history_stacked(res, out_dir, per_what="name") + plot_resource_history_stacked(res, out_dir, per_what="name", task_filter=args.filter_task) # per timeframe - plot_resource_history_stacked(res, out_dir, per_what="timeframe") + plot_resource_history_stacked(res, out_dir, per_what="timeframe", task_filter=args.filter_task) # per category - plot_resource_history_stacked(res, out_dir, per_what="category") + plot_resource_history_stacked(res, out_dir, per_what="category", task_filter=args.filter_task) # the following bar chart show the maximum resource needs for each task over all iterations From c56fca7646d71ba041a84e0fa9773b5e97d292ec Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Thu, 27 Jul 2023 11:13:13 +0200 Subject: [PATCH 0005/1239] [AnalysisQC] bool --> int in track-selection --- .../json/analysis-testing-TPCSkimming.json | 8 ++++---- .../analysis_testing/json/analysis-testing-data.json | 8 ++++---- MC/config/analysis_testing/json/analysis-testing-mc.json | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json b/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json index 3215662b7..5a73aeb23 100644 --- a/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json +++ b/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json @@ -254,13 +254,13 @@ }, "track-selection": { "compatibilityIU": "false", - "etaMax": "0.800000012", - "etaMin": "-0.800000012", + "etaMax": "0.8", + "etaMin": "-0.8", "isRun3": "true", "itsMatching": "1", - "produceFBextendedTable": "false", + "produceFBextendedTable": "0", "ptMax": "1e+10", - "ptMin": "0.100000001" + "ptMin": "0.1" }, "tree-writer-t-p-c-t-o-f": { "applyEvSel": "2", diff --git a/MC/config/analysis_testing/json/analysis-testing-data.json b/MC/config/analysis_testing/json/analysis-testing-data.json index 4e1ad8356..668854d2f 100644 --- a/MC/config/analysis_testing/json/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/analysis-testing-data.json @@ -263,13 +263,13 @@ }, "track-selection": { "compatibilityIU": "false", - "etaMax": "0.800000012", - "etaMin": "-0.800000012", + "etaMax": "0.8", + "etaMin": "-0.8", "isRun3": "true", "itsMatching": "1", - "produceFBextendedTable": "true", + "produceFBextendedTable": "-1", "ptMax": "1e+10", - "ptMin": "0.100000001" + "ptMin": "0.1" }, "emcal-correction-task": { "nonlinearityFunction": "DATA_TestbeamFinal_NoScale", diff --git a/MC/config/analysis_testing/json/analysis-testing-mc.json b/MC/config/analysis_testing/json/analysis-testing-mc.json index 1ca72ff0d..08f0c7701 100644 --- a/MC/config/analysis_testing/json/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/analysis-testing-mc.json @@ -1158,13 +1158,13 @@ }, "track-selection": { "compatibilityIU": "false", - "etaMax": "0.800000012", - "etaMin": "-0.800000012", + "etaMax": "0.8", + "etaMin": "-0.8", "isRun3": "true", "itsMatching": "1", - "produceFBextendedTable": "true", + "produceFBextendedTable": "-1", "ptMax": "1e+10", - "ptMin": "0.100000001" + "ptMin": "0.1" }, "emcal-correction-task": { "nonlinearityFunction": "MC_TestbeamFinal", From 8ff53e09a6b3e15ab9cdf687a81b76917118ab6c Mon Sep 17 00:00:00 2001 From: Raphaelle Bailhache Date: Fri, 28 Jul 2023 14:59:41 +0200 Subject: [PATCH 0006/1239] Tunning of MB event --- .../Generator_pythia8_GapTriggered_HFLepton.C | 2 +- .../pythia8/generator/pythia8_MB_gapevent.cfg | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 MC/config/PWGEM/pythia8/generator/pythia8_MB_gapevent.cfg diff --git a/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C b/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C index ce8dae943..2b90ca1b9 100644 --- a/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C +++ b/MC/config/PWGEM/external/generator/Generator_pythia8_GapTriggered_HFLepton.C @@ -28,7 +28,7 @@ public: auto seed = (gRandom->TRandom::GetSeed() % 900000000); cout<<"Initalizing extra PYTHIA object used to generate min-bias events..."<ExpandPathName("$O2DPG_ROOT/MC/config/common/pythia8/generator/pythia8_pp_cr2.cfg"); + TString pathconfigMB = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGEM/pythia8/generator/pythia8_MB_gapevent.cfg"); pythiaObjectMinimumBias.readFile(pathconfigMB.Data()); pythiaObjectMinimumBias.readString("Random:setSeed on"); pythiaObjectMinimumBias.readString("Random:seed " + std::to_string(seed)); diff --git a/MC/config/PWGEM/pythia8/generator/pythia8_MB_gapevent.cfg b/MC/config/PWGEM/pythia8/generator/pythia8_MB_gapevent.cfg new file mode 100644 index 000000000..82e7b979a --- /dev/null +++ b/MC/config/PWGEM/pythia8/generator/pythia8_MB_gapevent.cfg @@ -0,0 +1,14 @@ +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +SoftQCD:inelastic on # all inelastic processes + +### per default it is Monash anyway +Tune:pp = 14 + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. From 00228ca3f3ae5f51291767846c8598088b7a1e10 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Tue, 1 Aug 2023 00:47:28 +0200 Subject: [PATCH 0007/1239] Allow for margin at low pt Pt smearing can bias efficiency at the border of the acceptance --- MC/config/PWGLF/pythia8/generator/hypernuclei.gun | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun index 8ef9a9494..154cb401e 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun @@ -3,4 +3,4 @@ 1000010030 1 0.2 6 1000020030 1 0.2 6 1000020040 1 0.2 6 -1010010030 1 1 6 +1010010030 1 0.2 6 From d04c943a0990f17dec2a0e79d49e537ab67bb3c1 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Tue, 1 Aug 2023 11:06:50 +0200 Subject: [PATCH 0008/1239] [Sim Test] Relax criterion for transported particles --- test/common/kine_tests/test_generic_kine.C | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/test/common/kine_tests/test_generic_kine.C b/test/common/kine_tests/test_generic_kine.C index 3a2a9219c..dce63e93c 100644 --- a/test/common/kine_tests/test_generic_kine.C +++ b/test/common/kine_tests/test_generic_kine.C @@ -11,29 +11,26 @@ int test_generic_kine() auto tree = (TTree *)file.Get("o2sim"); std::vector *tracks{}; tree->SetBranchAddress("MCTrack", &tracks); - auto nEvents = tree->GetEntries(); + bool hasParticlesForTransport{}; - for (int i = 0; i < nEvents; i++) - { + for (int i = 0; i < tree->GetEntries();; i++) { tree->GetEntry(i); int iTrack{}; - int nToBeDone{}; - for (auto &track : *tracks) - { + for (auto &track : *tracks) { iTrack++; if (track.getToBeDone()) { - nToBeDone++; + hasParticlesForTransport = true; } if (!o2::mcgenstatus::isEncoded(track.getStatusCode())) { - std::cerr << "Track " << iTrack << " has invalid status encoding, make sure you set the status code correctly (see https://aliceo2group.github.io/simulation/docs/generators/).\n"; + std::cerr << "Particle " << iTrack << " has invalid status encoding, make sure you set the status code correctly (see https://aliceo2group.github.io/simulation/docs/generators/).\n"; return 1; } } - if (nToBeDone == 0) { - std::cerr << "Event " << i << " has no particles marked to be transported. Make sure they are marked correctly (see https://aliceo2group.github.io/simulation/docs/generators/).\n"; - return 1; - } + } + if (!hasParticlesForTransport) { + std::cerr << "No particles marked to be transported. Make sure they are marked correctly (see https://aliceo2group.github.io/simulation/docs/generators/).\n"; + return 1; } return 0; } From 5e5cfcbdec7fad5d3927d78f7c5c222228fa16c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Wed, 2 Aug 2023 17:34:50 +0200 Subject: [PATCH 0009/1239] Add triggered in general purpose injector generator (#1142) * Add option for gap triggered * Update runners * Update reso ini * Update nuclei ini * Add injection scheme for pp --- .../o2dpg_analysis_test_workflow.py | 2 +- MC/config/PWGLF/ini/GeneratorLFDeTrHe.ini | 9 +- .../PWGLF/ini/GeneratorLFStrangeness.ini | 17 +- .../ini/GeneratorLFStrangenessTriggered.ini | 11 + .../PWGLF/ini/GeneratorLF_Resonances_pp.ini | 11 +- .../PWGLF/ini/tests/GeneratorLFStrangeness.C | 52 +- .../tests/GeneratorLFStrangenessTriggered.C | 58 ++ .../ini/tests/GeneratorLF_Resonances_pp.C | 15 +- .../PWGLF/pythia8/decayer/strangeness.cfg | 1 + .../{particlelist.gun => nuclei.gun} | 0 .../PWGLF/pythia8/generator/nuclei_pp.gun | 9 + .../pythia8/generator/pythia8_inel_136tev.cfg | 18 + .../PWGLF/pythia8/generator_pythia8_LF.C | 540 ++++++++++++++++++ .../generator_pythia8_longlived_multiple.C | 313 ---------- MC/run/PWGLF/run_DeTrHeInjected.sh | 3 +- MC/run/PWGLF/run_StrangenessInjected.sh | 4 +- MC/run/PWGLF/run_StrangenessTriggered.sh | 44 ++ test/common/kine_tests/test_generic_kine.C | 2 +- 18 files changed, 762 insertions(+), 347 deletions(-) create mode 100644 MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini create mode 100644 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered.C create mode 100644 MC/config/PWGLF/pythia8/decayer/strangeness.cfg rename MC/config/PWGLF/pythia8/generator/{particlelist.gun => nuclei.gun} (100%) create mode 100644 MC/config/PWGLF/pythia8/generator/nuclei_pp.gun create mode 100644 MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg create mode 100644 MC/config/PWGLF/pythia8/generator_pythia8_LF.C delete mode 100644 MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C create mode 100755 MC/run/PWGLF/run_StrangenessTriggered.sh diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index be543831f..9ae7e0549 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -250,7 +250,7 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis piped_analysis = f" --configuration {configuration} | ".join(ana["tasks"]) piped_analysis += f" --configuration {configuration} --aod-file {input_aod}" if timeout is not None: - piped_analysis += f" --timeout {timeout}" + piped_analysis += f" --time-limit {timeout}" workflow.append(create_ana_task(ana["name"], piped_analysis, output_dir, needs=needs, is_mc=is_mc)) # append potential post-processing diff --git a/MC/config/PWGLF/ini/GeneratorLFDeTrHe.ini b/MC/config/PWGLF/ini/GeneratorLFDeTrHe.ini index 9ec909806..48c26d771 100644 --- a/MC/config/PWGLF/ini/GeneratorLFDeTrHe.ini +++ b/MC/config/PWGLF/ini/GeneratorLFDeTrHe.ini @@ -1,12 +1,13 @@ [GeneratorExternal] -fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C -funcName=generateLongLivedMultiple("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/particlelist.gun") -; funcName=generateLongLivedMultiple({{1000010020, 10, 0.2, 10}, {-1000010020, 10, 0.2, 10}, {1000010030, 10, 0.2, 10}, {-1000010030, 10, 0.2, 10}, {1000020030, 10, 0.2, 10}, {-1000020030, 10, 0.2, 10}}) +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.gun") +; funcName=generateLF({{1000010020, 10, 0.2, 10}, {-1000010020, 10, 0.2, 10}, {1000010030, 10, 0.2, 10}, {-1000010030, 10, 0.2, 10}, {1000020030, 10, 0.2, 10}, {-1000020030, 10, 0.2, 10}}) # Deuteron Anti-Deuteron Triton Anti-Triton Helium3 Anti-Helium3 [GeneratorPythia8] # config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg -config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.cfg +; config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.cfg [DecayerPythia8] config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg +config[1]=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangeness.ini b/MC/config/PWGLF/ini/GeneratorLFStrangeness.ini index 4e9d7e0df..c34778f53 100644 --- a/MC/config/PWGLF/ini/GeneratorLFStrangeness.ini +++ b/MC/config/PWGLF/ini/GeneratorLFStrangeness.ini @@ -1,12 +1,13 @@ [GeneratorExternal] -fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C -funcName=generateLongLivedMultiple("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun") -; funcName=generateLongLivedMultiple({{1000010020, 10, 0.2, 10}, {-1000010020, 10, 0.2, 10}, {1000010030, 10, 0.2, 10}, {-1000010030, 10, 0.2, 10}, {1000020030, 10, 0.2, 10}, {-1000020030, 10, 0.2, 10}}) -# Deuteron Anti-Deuteron Triton Anti-Triton Helium3 Anti-Helium3 +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", true, 0) +funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", true, 4) -[GeneratorPythia8] -# config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg -config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.cfg +; [GeneratorPythia8] # If injected the generator must be left empty +; config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg +; config=/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/decayer/strangeness.cfg -[DecayerPythia8] +[DecayerPythia8] # The only configuration must be given to the decayer to handle particles that are not handled in the transport code config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg +config[1]=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/decayer/strangeness.cfg +verbose=true \ No newline at end of file diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini new file mode 100644 index 000000000..d558289b0 --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini @@ -0,0 +1,11 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 0) +funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 4) + +[GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg + +[DecayerPythia8] +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini index 97d26eb87..073adad06 100644 --- a/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini +++ b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini @@ -1,9 +1,10 @@ [GeneratorExternal] -fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C -funcName=generateLongLivedMultiple("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonancelist.gun") +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonancelist.gun") -[GeneratorPythia8] # before for transport code! -config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonances.cfg +# [GeneratorPythia8] # before for transport code! +# config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonances.cfg [DecayerPythia8] # after for transport code! -config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg \ No newline at end of file +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg +config[1]=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonances.cfg \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangeness.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangeness.C index 49ba5d4e2..842ba854a 100644 --- a/MC/config/PWGLF/ini/tests/GeneratorLFStrangeness.C +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangeness.C @@ -1,12 +1,36 @@ int External() { std::string path{"o2sim_Kine.root"}; - int numberOfInjectedSignalsPerEvent{1}; - std::vector injectedPDGs = { - 3334, - -3334, - 3312, - -3312}; + // std::string path{"bkg_Kine.root"}; + std::vector numberOfInjectedSignalsPerEvent = {}; + std::vector injectedPDGs = {}; + std::string particleList = "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun"; + particleList = gSystem->ExpandPathName(particleList.c_str()); + + std::ifstream inputFile(particleList.c_str(), ios::in); + if (inputFile.is_open()) { + std::string l; + int n = 0; + while (getline(inputFile, l)) { + TString line = l; + line.Strip(TString::kBoth, ' '); + std::cout << n++ << " '" << line << "'" << std::endl; + if (line.IsNull() || line.IsWhitespace()) { + continue; + } + + if (line.BeginsWith("#")) { + std::cout << "Skipping\n"; + continue; + } + auto* arr = line.Tokenize(" "); + injectedPDGs.push_back(atoi(arr->At(0)->GetName())); + numberOfInjectedSignalsPerEvent.push_back(atoi(arr->At(1)->GetName())); + } + } else { + std::cout << "Cannot open file " << particleList << "\n"; + return 1; + } auto nInjection = injectedPDGs.size(); @@ -32,12 +56,18 @@ int External() auto nEvents = tree->GetEntries(); for (int i = 0; i < nEvents; i++) { auto check = tree->GetEntry(i); + std::cout << "Event " << i << "/" << tree->GetEntries() << std::endl; for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) { auto track = tracks->at(idxMCTrack); auto pdg = track.GetPdgCode(); auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); + std::cout << " particle " << idxMCTrack << " pdg: " << pdg << " getHepMCStatusCode " << getHepMCStatusCode(track.getStatusCode()) << " getGenStatusCode " << getGenStatusCode(track.getStatusCode()) << std::endl; + std::cout << " getMotherTrackId " << track.getMotherTrackId() << " getSecondMotherTrackId " << track.getSecondMotherTrackId() << " " << std::endl; int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG - if (it != injectedPDGs.end()) // found + if (!getHepMCStatusCode(track.getStatusCode())) { + continue; + } + if (it != injectedPDGs.end()) // found { // count signal PDG nSignal[index]++; @@ -47,12 +77,14 @@ int External() std::cout << "--------------------------------\n"; std::cout << "# Events: " << nEvents << "\n"; for (int i = 0; i < nInjection; i++) { - std::cout << "# Injected nuclei \n"; + std::cout << "# Injected particle \n"; std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; if (nSignal[i] == 0) { std::cerr << "No generated: " << injectedPDGs[i] << "\n"; - return 1; // At least one of the injected particles should be generated + // return 1; // At least one of the injected particles should be generated } } return 0; -} \ No newline at end of file +} + +void GeneratorLFStrangeness() { External(); } \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered.C new file mode 100644 index 000000000..49ba5d4e2 --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered.C @@ -0,0 +1,58 @@ +int External() +{ + std::string path{"o2sim_Kine.root"}; + int numberOfInjectedSignalsPerEvent{1}; + std::vector injectedPDGs = { + 3334, + -3334, + 3312, + -3312}; + + auto nInjection = injectedPDGs.size(); + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + if (!tree) { + std::cerr << "Cannot find tree o2sim in file " << path << "\n"; + return 1; + } + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + std::vector nSignal; + for (int i = 0; i < nInjection; i++) { + nSignal.push_back(0); + } + + auto nEvents = tree->GetEntries(); + for (int i = 0; i < nEvents; i++) { + auto check = tree->GetEntry(i); + for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) { + auto track = tracks->at(idxMCTrack); + auto pdg = track.GetPdgCode(); + auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); + int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG + if (it != injectedPDGs.end()) // found + { + // count signal PDG + nSignal[index]++; + } + } + } + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + for (int i = 0; i < nInjection; i++) { + std::cout << "# Injected nuclei \n"; + std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; + if (nSignal[i] == 0) { + std::cerr << "No generated: " << injectedPDGs[i] << "\n"; + return 1; // At least one of the injected particles should be generated + } + } + return 0; +} \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C index 0ad1a5b68..d87ec110c 100644 --- a/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C +++ b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C @@ -83,6 +83,7 @@ int External() nSignal.push_back(0); } std::vector> nDecays; + std::vector nNotDecayed; for (int i = 0; i < nInjection; i++) { std::vector nDecay; @@ -91,6 +92,7 @@ int External() nDecay.push_back(0); } nDecays.push_back(nDecay); + nNotDecayed.push_back(0); } auto nEvents = tree->GetEntries(); for (int i = 0; i < nEvents; i++) @@ -108,19 +110,26 @@ int External() nSignal[index]++; if(track.getFirstDaughterTrackId() < 0) { + nNotDecayed[index]++; continue; } for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { auto pdgDau = tracks->at(j).GetPdgCode(); + bool foundDau= false; // count decay PDGs for (int idxDaughter = 0; idxDaughter < decayDaughters[index].size(); ++idxDaughter) { if (pdgDau == decayDaughters[index][idxDaughter]) { nDecays[index][idxDaughter]++; + foundDau= true; + break; } } + if (!foundDau) { + std::cerr << "Decay daughter not found: " << pdg << " -> " << pdgDau << "\n"; + } } } } @@ -130,7 +139,7 @@ int External() for (int i = 0; i < nInjection; i++) { std::cout << "# Mother \n"; - std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; + std::cout << injectedPDGs[i] << " generated: " << nSignal[i] << ", " << nNotDecayed[i] << " did not decay\n"; if (nSignal[i] == 0){ std::cerr << "No generated: " << injectedPDGs[i] << "\n"; return 1; // At least one of the injected particles should be generated @@ -146,4 +155,6 @@ int External() } } return 0; -} \ No newline at end of file +} + +void GeneratorLF_Resonances_pp() { External(); } diff --git a/MC/config/PWGLF/pythia8/decayer/strangeness.cfg b/MC/config/PWGLF/pythia8/decayer/strangeness.cfg new file mode 100644 index 000000000..837da9962 --- /dev/null +++ b/MC/config/PWGLF/pythia8/decayer/strangeness.cfg @@ -0,0 +1 @@ +3334:onMode = true diff --git a/MC/config/PWGLF/pythia8/generator/particlelist.gun b/MC/config/PWGLF/pythia8/generator/nuclei.gun similarity index 100% rename from MC/config/PWGLF/pythia8/generator/particlelist.gun rename to MC/config/PWGLF/pythia8/generator/nuclei.gun diff --git a/MC/config/PWGLF/pythia8/generator/nuclei_pp.gun b/MC/config/PWGLF/pythia8/generator/nuclei_pp.gun new file mode 100644 index 000000000..697052a8d --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/nuclei_pp.gun @@ -0,0 +1,9 @@ +# PDG N ptMin ptMax genDecayed +1000010020 1 0.2 10 +-1000010020 1 0.2 10 +1000010030 1 0.2 10 +-1000010030 1 0.2 10 +1000020030 1 0.2 10 +-1000020030 1 0.2 10 +1000020040 1 0.2 10 +-1000020040 1 0.2 10 diff --git a/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg b/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg new file mode 100644 index 000000000..9f11de75b --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg @@ -0,0 +1,18 @@ +### beams +Beams:idA = 2212 # proton +Beams:idB = 2212 # proton +Beams:eCM = 13600. # GeV + +### processes +SoftQCD:inelastic = on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### phase space cuts +PhaseSpace:pTHatMin = 0.000000 +PhaseSpace:pTHatMax = -1.000000 + +Random:setSeed = on +Random:seed = 0 \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_LF.C b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C new file mode 100644 index 000000000..4428c94de --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C @@ -0,0 +1,540 @@ +/// +/// \file generator_pythia8_LF.C +/// \author Nicolò Jacazio nicolo.jacazio@cern.ch +/// \since 05/08/2022 +/// \brief Implementation of a gun generator for multiple particles, built on generator_pythia8_longlived.C +/// Needs PDG, Number of injected, minimum and maximum pT. These can be provided in three ways, bundeling variables, particles or from input file +/// usage: +/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_LF.C;GeneratorExternal.funcName=generateLF({1000010020, 1000010030}, {10, 10}, {0.5, 0.5}, {10, 10})'` +/// Here PDG, Number injected, pT limits are separated and matched by index +/// or: +/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_LF.C;GeneratorExternal.funcName=generateLF({{1000010020, 10, 0.5, 10}, {1000010030, 10, 0.5, 10}})'` +/// Here PDG, Number injected, pT limits are separated are divided per particle +/// or: +/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_LF.C;GeneratorExternal.funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.gun")'` +/// Here PDG, Number injected, pT limits are provided via an intermediate configuration file +/// + +#if !defined(__CLING__) || defined(__ROOTCLING__) +#include "SimulationDataFormat/MCGenStatus.h" +#include "SimulationDataFormat/MCUtils.h" +#include "fairlogger/Logger.h" +#include "TSystem.h" +#include +#include "Generators/GeneratorPythia8Param.h" +#include "Generators/DecayerPythia8Param.h" +#endif +#include "generator_pythia8_longlived.C" + +using namespace Pythia8; +using namespace o2::mcgenstatus; + +class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 +{ + public: + /// Parametric constructor + GeneratorPythia8LF(bool injOnePerEvent /*= true*/, + int gapBetweenInjection /*= 0*/, + bool useTrigger /*= false*/, + std::string pythiaCfgMb /*= "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/pythia8_inel_minbias.cfg"*/, + std::string pythiaCfgSignal /*= "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/pythia8_inel_signal.cfg"*/) : GeneratorPythia8{}, + mOneInjectionPerEvent{injOnePerEvent}, + mGapBetweenInjection{gapBetweenInjection}, + mUseTriggering{useTrigger} + { + LOG(info) << "GeneratorPythia8LF constructor"; + LOG(info) << "++ mOneInjectionPerEvent: " << mOneInjectionPerEvent; + LOG(info) << "++ mGapBetweenInjection: " << mGapBetweenInjection; + LOG(info) << "++ mUseTriggering: " << mUseTriggering; + LOG(info) << "++ pythiaCfgMb: " << pythiaCfgMb; + LOG(info) << "++ pythiaCfgSignal: " << pythiaCfgSignal; + gRandom->SetSeed(0); + if (useTrigger) { + mPythia.readString("ProcessLevel:all off"); + if (pythiaCfgMb == "") { // If no configuration file is provided, use the one from the Pythia8Param + auto& param = o2::eventgen::GeneratorPythia8Param::Instance(); + LOG(info) << "Instance LF \'Pythia8\' generator with following parameters for MB event"; + LOG(info) << param; + pythiaCfgMb = param.config; + } + if (pythiaCfgSignal == "") { // If no configuration file is provided, use the one from the Pythia8Param + auto& param = o2::eventgen::GeneratorPythia8Param::Instance(); + LOG(info) << "Instance LF \'Pythia8\' generator with following parameters for signal event"; + LOG(info) << param; + pythiaCfgSignal = param.config; + } + pythiaCfgMb = gSystem->ExpandPathName(pythiaCfgMb.c_str()); + pythiaCfgSignal = gSystem->ExpandPathName(pythiaCfgSignal.c_str()); + LOG(info) << " ++ Using trigger, initializing Pythia8 for trigger"; + if (!pythiaObjectMinimumBias.readFile(pythiaCfgMb)) { + LOG(fatal) << "Could not pythiaObjectMinimumBias.readFile(\"" << pythiaCfgMb << "\")"; + } + pythiaObjectMinimumBias.readString("Random:setSeed = on"); + pythiaObjectMinimumBias.readString("Random:seed =" + std::to_string(gRandom->Integer(900000000 - 2) + 1)); + + if (!pythiaObjectMinimumBias.init()) { + LOG(fatal) << "Could not pythiaObjectMinimumBias.init() from " << pythiaCfgMb; + } + if (!pythiaObjectSignal.readFile(pythiaCfgSignal)) { + LOG(fatal) << "Could not pythiaObjectSignal.readFile(\"" << pythiaCfgSignal << "\")"; + } + pythiaObjectSignal.readString("Random:setSeed = on"); + pythiaObjectSignal.readString("Random:seed =" + std::to_string(gRandom->Integer(900000000 - 2) + 1)); + if (!pythiaObjectSignal.init()) { + LOG(fatal) << "Could not pythiaObjectSignal.init() from " << pythiaCfgSignal; + } + } else { // Using simple injection with internal decay (if needed). Fetching the parameters from the configuration file of the PythiaDecayer + /** switch off process level **/ + mPythia.readString("ProcessLevel:all off"); + + /** config **/ + auto& paramGen = o2::eventgen::GeneratorPythia8Param::Instance(); + if (!paramGen.config.empty()) { + LOG(fatal) << "Configuration file provided for \'GeneratorPythia8\' should be empty for this injection scheme"; + return; + } + auto& param = o2::eventgen::DecayerPythia8Param::Instance(); + LOG(info) << "Init \'GeneratorPythia8LF\' with following parameters"; + LOG(info) << param; + for (int i = 0; i < 8; ++i) { + if (param.config[i].empty()) { + continue; + } + std::string config = gSystem->ExpandPathName(param.config[i].c_str()); + LOG(info) << "GeneratorPythia8LF Reading configuration from file: " << config; + if (!mPythia.readFile(config, true)) { + LOG(fatal) << "Failed to init \'DecayerPythia8\': problems with configuration file " + << config; + return; + } + } + + /** show changed particle data **/ + if (param.showChanged) { + mPythia.readString(std::string("Init:showChangedParticleData on")); + } else { + mPythia.readString(std::string("Init:showChangedParticleData off")); + } + + /** initialise **/ + if (!mPythia.init()) { + LOG(fatal) << "Failed to init \'DecayerPythia8\': init returned with error"; + return; + } + if (pythiaCfgSignal != "") { + LOG(fatal) << "Cannot use simple injection and have a configuration file. pythiaCfgSignal= `" << pythiaCfgSignal << "` must be empty"; + } + } + } + + /// Destructor + ~GeneratorPythia8LF() = default; + + //__________________________________________________________________ + Bool_t generateEvent() override + { + if (!mUseTriggering) { // If the triggering is used we handle the the gap when generating the signal + if (mGapBetweenInjection > 0) { + if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { + LOG(info) << "Skipping event " << mEventCounter; + return true; + } else if (mEventCounter % mGapBetweenInjection != 0) { + LOG(info) << "Skipping event " << mEventCounter; + return true; + } + } + } + LOG(info) << "generateEvent " << mEventCounter; + mPythia.event.reset(); + + mConfigToUse = mOneInjectionPerEvent ? static_cast(gRandom->Uniform(0.f, getNGuns())) : -1; + LOG(info) << "Using configuration " << mConfigToUse << " out of " << getNGuns() << ", of which " << mGunConfigs.size() << " are transport decayed and " << mGunConfigsGenDecayed.size() << " are generator decayed"; + + bool injectedForThisEvent = false; + int nConfig = mGunConfigs.size(); // We start counting from the configurations of the transport decayed particles + for (const ConfigContainer& cfg : mGunConfigsGenDecayed) { + nConfig++; + if (mConfigToUse >= 0 && (nConfig - 1) != mConfigToUse) { + continue; + } + LOG(info) << "Using config container "; + cfg.print(); + if (mUseTriggering) { // Do the triggering + bool doSignal = true; // Do signal or gap + if (mGapBetweenInjection > 0) { + if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { + doSignal = false; + } else if (mEventCounter % mGapBetweenInjection != 0) { + doSignal = false; + } + } + + if (doSignal) { + LOG(info) << "Generating triggered signal event for particle"; + cfg.print(); + bool satisfiesTrigger = false; + int nTries = 0; + while (!satisfiesTrigger) { + if (!pythiaObjectSignal.next()) { + continue; + } + //Check if triggered condition satisfied + for (Long_t j = 0; j < pythiaObjectSignal.event.size(); j++) { + const int& pypid = pythiaObjectSignal.event[j].id(); + const float& pyeta = pythiaObjectSignal.event[j].eta(); + const float& pypt = pythiaObjectSignal.event[j].pT(); + if (pypid == cfg.pdg && cfg.etaMin < pyeta && pyeta < cfg.etaMax && pypt > cfg.ptMin && pypt < cfg.ptMax) { + LOG(info) << "Found particle " << j << " " << pypid << " with eta " << pyeta << " and pT " << pypt << " in event " << mEventCounter << " after " << nTries << " tries"; + satisfiesTrigger = true; + break; + } + } + nTries++; + } + mPythia.event = pythiaObjectSignal.event; + } else { + LOG(info) << "Generating background event " << mEventCounter; + // Generate minimum-bias event + bool lGenerationOK = false; + while (!lGenerationOK) { + lGenerationOK = pythiaObjectMinimumBias.next(); + } + mPythia.event = pythiaObjectMinimumBias.event; + } + continue; + } + // Do the injection + for (int i{0}; i < cfg.nInject; ++i) { + const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); + const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); + const double phi = gRandom->Uniform(0, TMath::TwoPi()); + const double px{pt * std::cos(phi)}; + const double py{pt * std::sin(phi)}; + const double pz{pt * std::sinh(eta)}; + const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; + + Particle particle; + particle.id(cfg.pdg); + particle.status(11); + particle.m(cfg.mass); + particle.px(px); + particle.py(py); + particle.pz(pz); + particle.e(et); + particle.xProd(0.f); + particle.yProd(0.f); + particle.zProd(0.f); + mPythia.particleData.mayDecay(cfg.pdg, true); // force decay + mPythia.event.append(particle); + } + injectedForThisEvent = true; + } + if (injectedForThisEvent) { + LOG(info) << "Calling next!"; + mPythia.moreDecays(); + mPythia.next(); + if (mPythia.event.size() <= 2) { + LOG(fatal) << "Event size is " << mPythia.event.size() << ", this is not good! Check that the decay actually happened or consider not using the generator decayed particles!"; + } else { + LOG(info) << "Event size is " << mPythia.event.size() << " particles"; + } + } + + if (mVerbose) { + LOG(info) << "Eventlisting"; + mPythia.event.list(1); + mPythia.stat(); + } + return true; + } + + //__________________________________________________________________ + Bool_t importParticles() override + { + if (!mUseTriggering) { // If the triggering is used we handle the the gap when generating the signal + if (mGapBetweenInjection > 0) { + if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { + LOG(info) << "Skipping importParticles event " << mEventCounter++; + return true; + } else if (mEventCounter % mGapBetweenInjection != 0) { + LOG(info) << "Skipping importParticles event " << mEventCounter++; + return true; + } + } + } + LOG(info) << "importParticles " << mEventCounter++; + GeneratorPythia8::importParticles(); + int nConfig = 0; + for (const ConfigContainer& cfg : mGunConfigs) { + nConfig++; + if (mConfigToUse >= 0 && (nConfig - 1) != mConfigToUse) { + continue; + } + LOGF(info, "Injecting %i particles with PDG %i, pT in [%f, %f]", cfg.nInject, cfg.pdg, cfg.ptMin, cfg.ptMax); + + for (int i{0}; i < cfg.nInject; ++i) { + const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); + const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); + const double phi = gRandom->Uniform(0, TMath::TwoPi()); + const double px{pt * std::cos(phi)}; + const double py{pt * std::sin(phi)}; + const double pz{pt * std::sinh(eta)}; + const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; + + // TParticle::TParticle(Int_t pdg, + // Int_t status, + // Int_t mother1, Int_t mother2, + // Int_t daughter1, Int_t daughter2, + // Double_t px, Double_t py, Double_t pz, Double_t etot, + // Double_t vx, Double_t vy, Double_t vz, Double_t time) + + mParticles.push_back(TParticle(cfg.pdg, + MCGenStatusEncoding(1, 1).fullEncoding, + -1, -1, + -1, -1, + px, py, pz, et, + 0., 0., 0., 0.)); + // make sure status code is encoded properly. Transport flag will be set by default and we have nothing + // to do since all pushed particles should be tracked. + o2::mcutils::MCGenHelper::encodeParticleStatusAndTracking(mParticles.back()); + } + nConfig++; + } + if (mVerbose) { + LOG(info) << "Printing particles that are appended"; + int n = 0; + for (const auto& p : mParticles) { + LOG(info) << "Particle " << n++ << " is a " << p.GetPdgCode() << " with status " << p.GetStatusCode() << " and px = " << p.Px() << " py = " << p.Py() << " pz = " << p.Pz(); + } + } + return true; + } + + struct ConfigContainer { + ConfigContainer(int input_pdg = 0, int n = 1, float p = 1, float P = 10) : pdg{input_pdg}, + nInject{n}, + ptMin{p}, + ptMax{P} + { + mass = GeneratorPythia8LongLivedGun::getMass(pdg); + if (mass <= 0) { + LOG(fatal) << "Could not find mass for pdg " << pdg; + } + LOGF(info, "ConfigContainer: pdg = %i, nInject = %i, ptMin = %f, ptMax = %f, mass = %f", pdg, nInject, ptMin, ptMax, mass); + }; + ConfigContainer(TObjArray* arr) : ConfigContainer(atoi(arr->At(0)->GetName()), + atoi(arr->At(1)->GetName()), + atof(arr->At(2)->GetName()), + atof(arr->At(3)->GetName())){}; + + int pdg = 0; + int nInject = 1; + float ptMin = 1; + float ptMax = 10; + float etaMin = -1.f; + float etaMax = 1.f; + double mass = 0.f; + void print() const + { + LOGF(info, "int pdg = %i", pdg); + LOGF(info, "int nInject = %i", nInject); + LOGF(info, "float ptMin = %f", ptMin); + LOGF(info, "float ptMax = %f", ptMax); + LOGF(info, "float etaMin = %f", etaMin); + LOGF(info, "float etaMax = %f", etaMax); + LOGF(info, "double mass = %f", mass); + } + }; + + //__________________________________________________________________ + ConfigContainer addGun(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) + { + if (mUseTriggering) { // If in trigger mode, every particle needs to be generated from pythia + return addGunGenDecayed(input_pdg, nInject, ptMin, ptMax); + } + ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; + mGunConfigs.push_back(cfg); + return cfg; + } + + //__________________________________________________________________ + ConfigContainer addGun(ConfigContainer cfg) { return addGun(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } + + //__________________________________________________________________ + ConfigContainer addGunGenDecayed(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) + { + ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; + mGunConfigsGenDecayed.push_back(cfg); + return cfg; + } + + //__________________________________________________________________ + ConfigContainer addGunGenDecayed(ConfigContainer cfg) { return addGunGenDecayed(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } + + //__________________________________________________________________ + long int getNGuns() const { return mGunConfigs.size() + mGunConfigsGenDecayed.size(); } + + //__________________________________________________________________ + void print() + { + LOG(info) << "GeneratorPythia8LF configuration with " << getNGuns() << " guns:"; + LOG(info) << "Particles decayed by the transport:"; + int n = 0; + for (const auto& cfg : mGunConfigs) { + LOG(info) << n++ << "/" << mGunConfigs.size() << ":"; + cfg.print(); + } + n = 0; + LOG(info) << "Particles decayed by the generator:"; + for (const auto& cfg : mGunConfigsGenDecayed) { + LOG(info) << n++ << "/" << mGunConfigs.size() << ":"; + cfg.print(); + } + } + + void setVerbose(bool verbose = true) { mVerbose = verbose; } + + private: + // Configuration + const bool mOneInjectionPerEvent = true; // if true, only one injection per event is performed, i.e. if multiple PDG (including antiparticles) are requested to be injected only one will be done per event + const bool mUseTriggering = false; // if true, use triggering instead of injection + const int mGapBetweenInjection = 0; // Gap between two signal events. 0 means injection at every event + + // Running variables + int mConfigToUse = -1; // Index of the configuration to use + int mEventCounter = 0; // Event counter + bool mVerbose = true; // Verbosity flag + + std::vector mGunConfigs; // List of gun configurations to use + std::vector mGunConfigsGenDecayed; // List of gun configurations to use that will be decayed by the generator + Pythia pythiaObjectSignal; // Signal collision generator + Pythia pythiaObjectMinimumBias; // Minimum bias collision generator +}; + +///___________________________________________________________ +/// Create generator via arrays of entries. By default injecting in every event and all particles +FairGenerator* generateLF(std::vector PDGs, std::vector nInject, std::vector ptMin, std::vector ptMax) +{ + const std::vector entries = {PDGs.size(), nInject.size(), ptMin.size(), ptMax.size()}; + if (!std::equal(entries.begin() + 1, entries.end(), entries.begin())) { + LOGF(fatal, "Not equal number of entries, check configuration"); + return nullptr; + } + GeneratorPythia8LF* multiGun = new GeneratorPythia8LF(false, 0, false, "", ""); + for (unsigned long i = 0; i < entries[0]; i++) { + multiGun->addGun(PDGs[i], nInject[i], ptMin[i], ptMax[i]); + } + return multiGun; +} + +///___________________________________________________________ +/// Create generator via an array of configurations +FairGenerator* generateLF(std::vector cfg, + std::vector cfgGenDecayed, + bool injectOnePDGPerEvent = true, + int gapBetweenInjection = 0, + bool useTrigger = false, + std::string pythiaCfgMb = "", + std::string pythiaCfgSignal = "") +{ + GeneratorPythia8LF* multiGun = new GeneratorPythia8LF(injectOnePDGPerEvent, gapBetweenInjection, useTrigger, pythiaCfgMb, pythiaCfgSignal); + for (const auto& c : cfg) { + LOGF(info, "Adding gun %i", multiGun->getNGuns()); + c.print(); + multiGun->addGun(c); + } + for (const auto& c : cfgGenDecayed) { + LOGF(info, "Adding gun %i, particle will be decayed by the generator", multiGun->getNGuns()); + c.print(); + multiGun->addGunGenDecayed(c); + } + multiGun->print(); + return multiGun; +} + +///___________________________________________________________ +/// Create generator via input file +FairGenerator* generateLF(std::string configuration = "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.gun", + bool injectOnePDGPerEvent = true, + int gapBetweenInjection = 0, + bool useTrigger = false, + std::string pythiaCfgMb = "", + std::string pythiaCfgSignal = "") +{ + configuration = gSystem->ExpandPathName(configuration.c_str()); + LOGF(info, "Using configuration file '%s'", configuration.c_str()); + std::ifstream inputFile(configuration.c_str(), ios::in); + std::vector cfgVec; + std::vector cfgVecGenDecayed; + if (inputFile.is_open()) { + std::string l; + int n = 0; + while (getline(inputFile, l)) { + TString line = l; + line.Strip(TString::kBoth, ' '); + std::cout << n++ << " '" << line << "'" << endl; + if (line.IsNull() || line.IsWhitespace()) { + continue; + } + + if (line.BeginsWith("#")) { + std::cout << "Skipping\n"; + continue; + } + if (line.Contains("genDecayed")) { + cfgVecGenDecayed.push_back(GeneratorPythia8LF::ConfigContainer{line.Tokenize(" ")}); + } else { + cfgVec.push_back(GeneratorPythia8LF::ConfigContainer{line.Tokenize(" ")}); + } + } + } else { + LOGF(fatal, "Can't open '%s' !", configuration.c_str()); + return nullptr; + } + return generateLF(cfgVec, cfgVecGenDecayed, injectOnePDGPerEvent, gapBetweenInjection, useTrigger, pythiaCfgMb, pythiaCfgSignal); +} + +///___________________________________________________________ +/// Create generator via input file for the triggered mode +FairGenerator* generateLFTriggered(std::string configuration = "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei.gun", + int gapBetweenInjection = 0, + std::string pythiaCfgMb = "", + std::string pythiaCfgSignal = "") +{ + return generateLF(configuration, /*injectOnePDGPerEvent=*/true, gapBetweenInjection, /*useTrigger=*/true, pythiaCfgMb, pythiaCfgSignal); +} + +///___________________________________________________________ +void generator_pythia8_LF(bool testInj = true, bool testTrg = false) +{ + LOG(info) << "Compiled correctly!"; + if (!testInj && !testTrg) { + return; + } + // Injected mode + if (testInj) { + LOG(info) << "Testing the injected mode"; + auto* gen = static_cast(generateLF("/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun")); + gen->setVerbose(); + gen->Print(); + gen->print(); + gen->Init(); + gen->generateEvent(); + gen->importParticles(); + } + + // Triggered mode + if (testTrg) { + LOG(info) << "Testing the triggered mode"; + GeneratorPythia8LF* gen = static_cast(generateLFTriggered("/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", + /*gapBetweenInjection=*/0, + /*pythiaCfgMb=*/"/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/inel136tev.cfg", + /*pythiaCfgSignal=*/"/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/inel136tev.cfg")); + gen->setVerbose(); + gen->Print(); + gen->print(); + gen->Init(); + gen->generateEvent(); + gen->importParticles(); + } +} diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C deleted file mode 100644 index 63bb5ede7..000000000 --- a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_multiple.C +++ /dev/null @@ -1,313 +0,0 @@ -/// -/// \file generator_pythia8_longlived_multiple.C -/// \author Nicolò Jacazio nicolo.jacazio@cern.ch -/// \since 05/08/2022 -/// \brief Implementation of a gun generator for multiple particles, built on generator_pythia8_longlived.C -/// Needs PDG, Number of injected, minimum and maximum pT. These can be provided in three ways, bundeling variables, particles or from input file -/// usage: -/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_longlived_multiple.C;GeneratorExternal.funcName=generateLongLivedMultiple({1000010020, 1000010030}, {10, 10}, {0.5, 0.5}, {10, 10})'` -/// Here PDG, Number injected, pT limits are separated and matched by index -/// or: -/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_longlived_multiple.C;GeneratorExternal.funcName=generateLongLivedMultiple({{1000010020, 10, 0.5, 10}, {1000010030, 10, 0.5, 10}})'` -/// Here PDG, Number injected, pT limits are separated are divided per particle -/// or: -/// `o2-sim -g external --configKeyValues 'GeneratorExternal.fileName=generator_pythia8_longlived_multiple.C;GeneratorExternal.funcName=generateLongLivedMultiple("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/particlelist.gun")'` -/// Here PDG, Number injected, pT limits are provided via an intermediate configuration file -/// - -#if !defined(__CLING__) || defined(__ROOTCLING__) -#include "SimulationDataFormat/MCGenStatus.h" -#include "SimulationDataFormat/MCUtils.h" -#include "fairlogger/Logger.h" -#include "TSystem.h" -#include -#endif -#include "generator_pythia8_longlived.C" - -using namespace Pythia8; -using namespace o2::mcgenstatus; - -class GeneratorPythia8LongLivedGunMultiple : public GeneratorPythia8LongLivedGun -{ - public: - /// constructor - GeneratorPythia8LongLivedGunMultiple(bool injOnePerEvent /*= true*/, int gapBetweenInjection /*= 0*/) : GeneratorPythia8LongLivedGun{0}, mOneInjectionPerEvent{injOnePerEvent}, mGapBetweenInjection{gapBetweenInjection} - { - } - - /// Destructor - ~GeneratorPythia8LongLivedGunMultiple() = default; - - int mConfigToUse = -1; - int mEventCounter = 0; - int mGapBetweenInjection = 0; // Gap between two signal events. 0 means injection at every event - //__________________________________________________________________ - Bool_t generateEvent() override - { - if (mGapBetweenInjection > 0) { - if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { - LOG(info) << "Skipping event " << mEventCounter; - return true; - } else if (mEventCounter % mGapBetweenInjection != 0) { - LOG(info) << "Skipping event " << mEventCounter; - return true; - } - } - LOG(info) << "generateEvent " << mEventCounter; - mPythia.event.reset(); - - mConfigToUse = mOneInjectionPerEvent ? static_cast(gRandom->Uniform(0.f, getNGuns())) : -1; - LOGF(info, "Using configuration %i out of %lli, %lli transport decayed, %lli generator decayed", mConfigToUse, getNGuns(), mGunConfigs.size(), mGunConfigsGenDecayed.size()); - - int nConfig = mGunConfigs.size(); // We start counting from the configurations of the transport decayed particles - for (const ConfigContainer& cfg : mGunConfigsGenDecayed) { - nConfig++; - if (mConfigToUse >= 0 && (nConfig - 1) != mConfigToUse) { - continue; - } - - for (int i{0}; i < cfg.nInject; ++i) { - const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); - const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); - const double phi = gRandom->Uniform(0, TMath::TwoPi()); - const double px{pt * std::cos(phi)}; - const double py{pt * std::sin(phi)}; - const double pz{pt * std::sinh(eta)}; - const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; - - Particle particle; - particle.id(cfg.pdg); - particle.status(MCGenStatusEncoding(1, 1).fullEncoding); - particle.status(11); - particle.m(cfg.mass); - particle.px(px); - particle.py(py); - particle.pz(pz); - particle.e(et); - particle.xProd(0.f); - particle.yProd(0.f); - particle.zProd(0.f); - LOG(info) << "Appending particle " << i << "/" << cfg.nInject - 1 << " pdg = " << particle.id() << " at position " << mPythia.event.append(particle) << "/" << mPythia.event.size(); - } - } - mPythia.next(); - LOG(info) << "Eventlisting"; - mPythia.event.list(1); - mPythia.stat(); - return true; - } - - //__________________________________________________________________ - Bool_t importParticles() override - { - if (mGapBetweenInjection > 0) { - if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { - LOG(info) << "Skipping importParticles event " << mEventCounter++; - return true; - } else if (mEventCounter % mGapBetweenInjection != 0) { - LOG(info) << "Skipping importParticles event " << mEventCounter++; - return true; - } - } - LOG(info) << "importParticles " << mEventCounter++; - GeneratorPythia8::importParticles(); - int nConfig = 0; - for (const ConfigContainer& cfg : mGunConfigs) { - nConfig++; - if (mConfigToUse >= 0 && (nConfig - 1) != mConfigToUse) { - continue; - } - LOGF(info, "Injecting %i particles with PDG %i, pT in [%f, %f]", cfg.nInject, cfg.pdg, cfg.ptMin, cfg.ptMax); - - for (int i{0}; i < cfg.nInject; ++i) { - const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); - const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); - const double phi = gRandom->Uniform(0, TMath::TwoPi()); - const double px{pt * std::cos(phi)}; - const double py{pt * std::sin(phi)}; - const double pz{pt * std::sinh(eta)}; - const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; - - // TParticle::TParticle(Int_t pdg, - // Int_t status, - // Int_t mother1, Int_t mother2, - // Int_t daughter1, Int_t daughter2, - // Double_t px, Double_t py, Double_t pz, Double_t etot, - // Double_t vx, Double_t vy, Double_t vz, Double_t time) - - mParticles.push_back(TParticle(cfg.pdg, - MCGenStatusEncoding(1, 1).fullEncoding, - -1, -1, - -1, -1, - px, py, pz, et, - 0., 0., 0., 0.)); - // make sure status code is encoded properly. Transport flag will be set by default and we have nothing - // to do since all pushed particles should be tracked. - o2::mcutils::MCGenHelper::encodeParticleStatusAndTracking(mParticles.back()); - } - nConfig++; - } - return true; - } - - struct ConfigContainer { - ConfigContainer(int input_pdg = 0, int n = 1, float p = 1, float P = 10) : pdg{input_pdg}, - nInject{n}, - ptMin{p}, - ptMax{P} - { - mass = GeneratorPythia8LongLivedGun::getMass(pdg); - LOGF(info, "ConfigContainer: pdg = %i, nInject = %i, ptMin = %f, ptMax = %f, mass = %f", pdg, nInject, ptMin, ptMax, mass); - }; - ConfigContainer(TObjArray* arr) : ConfigContainer(atoi(arr->At(0)->GetName()), - atoi(arr->At(1)->GetName()), - atof(arr->At(2)->GetName()), - atof(arr->At(3)->GetName())){}; - - int pdg = 0; - int nInject = 1; - float ptMin = 1; - float ptMax = 10; - float etaMin = -1.f; - float etaMax = 1.f; - double mass = 0.f; - void print() const - { - LOGF(info, "int pdg = %i", pdg); - LOGF(info, "int nInject = %i", nInject); - LOGF(info, "float ptMin = %f", ptMin); - LOGF(info, "float ptMax = %f", ptMax); - LOGF(info, "float etaMin = %f", etaMin); - LOGF(info, "float etaMax = %f", etaMax); - LOGF(info, "double mass = %f", mass); - } - }; - - //__________________________________________________________________ - ConfigContainer addGun(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) - { - ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; - mGunConfigs.push_back(cfg); - return cfg; - } - - //__________________________________________________________________ - ConfigContainer addGun(ConfigContainer cfg) { return addGun(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } - - //__________________________________________________________________ - ConfigContainer addGunGenDecayed(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) - { - ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; - mGunConfigsGenDecayed.push_back(cfg); - return cfg; - } - - //__________________________________________________________________ - ConfigContainer addGunGenDecayed(ConfigContainer cfg) { return addGunGenDecayed(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } - - //__________________________________________________________________ - long int getNGuns() const { return mGunConfigs.size() + mGunConfigsGenDecayed.size(); } - - //__________________________________________________________________ - void print() - { - LOG(info) << "GeneratorPythia8LongLivedGunMultiple configuration with " << getNGuns() << " guns:"; - LOG(info) << "Particles decayed by the transport:"; - for (const auto& cfg : mGunConfigs) { - cfg.print(); - } - LOG(info) << "Particles decayed by the generator:"; - for (const auto& cfg : mGunConfigsGenDecayed) { - cfg.print(); - } - } - - private: - const bool mOneInjectionPerEvent = true; // if true, only one injection per event is performed, i.e. if multiple PDG (including antiparticles) are requested to be injected only one will be done per event - std::vector mGunConfigs; // List of gun configurations to use - std::vector mGunConfigsGenDecayed; // List of gun configurations to use that will be decayed by the generator -}; - -///___________________________________________________________ -/// Create generator via arrays of entries. By default injecting in every event and all particles -FairGenerator* generateLongLivedMultiple(std::vector PDGs, std::vector nInject, std::vector ptMin, std::vector ptMax) -{ - const std::vector entries = {PDGs.size(), nInject.size(), ptMin.size(), ptMax.size()}; - if (!std::equal(entries.begin() + 1, entries.end(), entries.begin())) { - LOGF(fatal, "Not equal number of entries, check configuration"); - return nullptr; - } - GeneratorPythia8LongLivedGunMultiple* multiGun = new GeneratorPythia8LongLivedGunMultiple(false, 0); - for (unsigned long i = 0; i < entries[0]; i++) { - multiGun->addGun(PDGs[i], nInject[i], ptMin[i], ptMax[i]); - } - return multiGun; -} - -///___________________________________________________________ -/// Create generator via an array of configurations -FairGenerator* generateLongLivedMultiple(std::vector cfg, - std::vector cfgGenDecayed, - bool injectOnePDGPerEvent = true, - int gapBetweenInjection = 0) -{ - GeneratorPythia8LongLivedGunMultiple* multiGun = new GeneratorPythia8LongLivedGunMultiple(injectOnePDGPerEvent, gapBetweenInjection); - for (const auto& c : cfg) { - LOGF(info, "Adding gun %i", multiGun->getNGuns()); - c.print(); - multiGun->addGun(c); - } - for (const auto& c : cfgGenDecayed) { - LOGF(info, "Adding gun %i, particle will be decayed by the generator", multiGun->getNGuns()); - c.print(); - multiGun->addGunGenDecayed(c); - } - multiGun->print(); - return multiGun; -} - -///___________________________________________________________ -/// Create generator via input file -FairGenerator* generateLongLivedMultiple(std::string configuration = "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/particlelist.gun", - bool injectOnePDGPerEvent = true, - int gapBetweenInjection = 0) -{ - configuration = gSystem->ExpandPathName(configuration.c_str()); - LOGF(info, "Using configuration file '%s'", configuration.c_str()); - std::ifstream inputFile(configuration.c_str(), ios::in); - std::vector cfgVec; - std::vector cfgVecGenDecayed; - if (inputFile.is_open()) { - std::string l; - int n = 0; - while (getline(inputFile, l)) { - TString line = l; - line.Strip(TString::kBoth, ' '); - std::cout << n++ << " '" << line << "'" << endl; - if (line.IsNull() || line.IsWhitespace()) { - continue; - } - - if (line.BeginsWith("#")) { - std::cout << "Skipping\n"; - continue; - } - if (line.Contains("genDecayed")) { - cfgVecGenDecayed.push_back(GeneratorPythia8LongLivedGunMultiple::ConfigContainer{line.Tokenize(" ")}); - } else { - cfgVec.push_back(GeneratorPythia8LongLivedGunMultiple::ConfigContainer{line.Tokenize(" ")}); - } - } - } else { - LOGF(fatal, "Can't open '%s' !", configuration.c_str()); - return nullptr; - } - return generateLongLivedMultiple(cfgVec, cfgVecGenDecayed, injectOnePDGPerEvent, gapBetweenInjection); -} - -///___________________________________________________________ -/// Create generator via input file injecting all particles -FairGenerator* generateLongLivedMultipleInjectInAllEvents(std::string configuration = "${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/particlelist.gun") { return generateLongLivedMultiple(configuration, false); } - -///___________________________________________________________ -void generator_pythia8_longlived_multiple() { Printf("Compiled correctly!"); } diff --git a/MC/run/PWGLF/run_DeTrHeInjected.sh b/MC/run/PWGLF/run_DeTrHeInjected.sh index 6a003d24f..439b505bc 100755 --- a/MC/run/PWGLF/run_DeTrHeInjected.sh +++ b/MC/run/PWGLF/run_DeTrHeInjected.sh @@ -36,7 +36,8 @@ $O2_SIM_WORKFLOW -eCM ${ENERGY} -col ${SYSTEM} -gen external \ -j ${NWORKERS} \ -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} \ -confKey "Diamond.width[2]=6." \ - ${SEED} -mod "${MODULES}" \ + ${SEED} \ + -procBkg inel -colBkg $SYSTEM --embedding -nb ${NBKGEVENTS} -genBkg pythia8 \ -e ${SIMENGINE} \ -ini $CFGINIFILE diff --git a/MC/run/PWGLF/run_StrangenessInjected.sh b/MC/run/PWGLF/run_StrangenessInjected.sh index 8a2ee738b..9c07c9271 100755 --- a/MC/run/PWGLF/run_StrangenessInjected.sh +++ b/MC/run/PWGLF/run_StrangenessInjected.sh @@ -13,13 +13,13 @@ export IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE=1 #export ALICEO2_CCDB_LOCALCACHE=.ccdb # ----------- LOAD UTILITY FUNCTIONS -------------------------- -. ${O2_ROOT}/share/scripts/jobutils.sh +. ${O2_ROOT}/share/scripts/jobutils.sh # ----------- START ACTUAL JOB ----------------------------- NWORKERS=${NWORKERS:-8} SIMENGINE=${SIMENGINE:-TGeant4} -NSIGEVENTS=${NSIGEVENTS:-2} +NSIGEVENTS=${NSIGEVENTS:-1} NBKGEVENTS=${NBKGEVENTS:-1} NTIMEFRAMES=${NTIMEFRAMES:-1} INTRATE=${INTRATE:-50000} diff --git a/MC/run/PWGLF/run_StrangenessTriggered.sh b/MC/run/PWGLF/run_StrangenessTriggered.sh new file mode 100755 index 000000000..412edf995 --- /dev/null +++ b/MC/run/PWGLF/run_StrangenessTriggered.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- CONFIGURE -------------------------- +export IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE=1 +#export ALICEO2_CCDB_LOCALCACHE=.ccdb + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +SIMENGINE=${SIMENGINE:-TGeant4} +NSIGEVENTS=${NSIGEVENTS:-10} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-50000} +SYSTEM=${SYSTEM:-pp} +ENERGY=${ENERGY:-13600} +CFGINIFILE=${CFGINIFILE:-"${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini"} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +echo "NWORKERS = $NWORKERS" + +# create workflow +O2_SIM_WORKFLOW=${O2_SIM_WORKFLOW:-"${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py"} +$O2_SIM_WORKFLOW -eCM ${ENERGY} -col ${SYSTEM} -gen external \ + -j ${NWORKERS} \ + -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} \ + -confKey "Diamond.width[2]=6." \ + ${SEED} \ + -e ${SIMENGINE} \ + -ini $CFGINIFILE + +# run workflow +O2_SIM_WORKFLOW_RUNNER=${O2_SIM_WORKFLOW_RUNNER:-"${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py"} +$O2_SIM_WORKFLOW_RUNNER -f workflow.json -tt aod --cpu-limit $NWORKERS diff --git a/test/common/kine_tests/test_generic_kine.C b/test/common/kine_tests/test_generic_kine.C index dce63e93c..fcad179ec 100644 --- a/test/common/kine_tests/test_generic_kine.C +++ b/test/common/kine_tests/test_generic_kine.C @@ -13,7 +13,7 @@ int test_generic_kine() tree->SetBranchAddress("MCTrack", &tracks); bool hasParticlesForTransport{}; - for (int i = 0; i < tree->GetEntries();; i++) { + for (int i = 0; i < tree->GetEntries(); i++) { tree->GetEntry(i); int iTrack{}; for (auto &track : *tracks) { From 04a3414003ab061b78d6eee9b2596938a258468a Mon Sep 17 00:00:00 2001 From: Francesco Noferini Date: Thu, 3 Aug 2023 18:17:22 +0200 Subject: [PATCH 0010/1239] set ft0-eff with energy/coll_system (#1159) --- MC/bin/o2dpg_sim_workflow_anchored.py | 21 ++++++++++++++++++--- MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh | 4 ++-- MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh | 4 ++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 0f64536d0..6157020bc 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -275,8 +275,10 @@ def main(): parser.add_argument("--split-id", type=int, help="The split id of this job within the whole production --prod-split)", default=0) parser.add_argument("-tf", type=int, help="number of timeframes per job", default=1) parser.add_argument("--ccdb-IRate", type=bool, help="whether to try fetching IRate from CCDB/CTP", default=True) - parser.add_argument("--ft0-eff", type=float, dest="ft0_eff", help="FT0 eff needed for IR", default=0.759) + parser.add_argument("--ft0-eff", type=float, dest="ft0_eff", help="FT0 eff needed for IR", default=-1.0) parser.add_argument('forward', nargs=argparse.REMAINDER) # forward args passed to actual workflow creation + parser.add_argument("-eCM", type=float, dest="eCM", help="Energy", default=13600) + parser.add_argument("-col", type=str, dest="col", help="Collision System", default="pp") args = parser.parse_args() # split id should not be larger than production id @@ -307,8 +309,21 @@ def main(): forwardargs = " ".join([ a for a in args.forward if a != '--' ]) # retrieve interaction rate rate = None + if args.ccdb_IRate == True: - rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., args.ft0_eff) + effT0 = args.ft0_eff + if effT0 < 0: + if args.col == "pp": + if args.eCM > 5000: + effT0 = 0.759 + else: + effT0 = 0.68 + elif args.col == "PbPb": + effT0 = 4.0 + else: + effT0 = 0.759 + + rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effT0) if rate != None: # if the rate calculation was successful we will use it, otherwise we fall back to some rate given as part @@ -321,7 +336,7 @@ def main(): # we finally pass forward to the unanchored MC workflow creation # TODO: this needs to be done in a pythonic way clearly - forwardargs += " -tf " + str(args.tf) + " --sor " + str(GLOparams["SOR"]) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(GLOparams["FirstOrbit"]) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + forwardargs += " -tf " + str(args.tf) + " --sor " + str(GLOparams["SOR"]) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(GLOparams["FirstOrbit"]) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(args.col) + " -eCM " + str(args.eCM) print ("forward args ", forwardargs) cmd = "${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py " + forwardargs print ("Creating time-anchored workflow...") diff --git a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh index b7d033fad..7a5ca1e4e 100755 --- a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh +++ b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh @@ -99,10 +99,10 @@ NSIGEVENTS=${NSIGEVENTS:-22} # THIS NEEDS TO COME FROM OUTSIDE # echo "$" | awk -F' -- ' '{print $1, $3}' -baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER}" +baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" # THIS NEEDS TO COME FROM OUTSIDE -remainingargs="-eCM 900 -col pp -gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ +remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ --include-local-qc --include-analysis --mft-reco-full" diff --git a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh index 105454641..814b832c2 100755 --- a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh +++ b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh @@ -89,10 +89,10 @@ NSIGEVENTS=${NSIGEVENTS:-22} # THIS NEEDS TO COME FROM OUTSIDE # echo "$" | awk -F' -- ' '{print $1, $3}' -baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER}" +baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" # THIS NEEDS TO COME FROM OUTSIDE -remainingargs="-eCM 900 -col pp -gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ +remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ --include-local-qc --include-analysis --mft-reco-full" From e30e79e28073dd424a9313bb839c2ae4b55bb568 Mon Sep 17 00:00:00 2001 From: Luca Barioglio Date: Fri, 4 Aug 2023 15:08:48 +0200 Subject: [PATCH 0011/1239] Set rapidity instead of pseudorapidity (#1160) * Inject in rapidity instead of pseudorapidity * Correct method name --- ...generator_pythia8_longlived_gaptriggered.C | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C index ebab0c18c..ee96fe3aa 100644 --- a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +++ b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C @@ -80,11 +80,11 @@ public: } } - /// Set pseudorapidity - void setEta(float eta_min, float eta_max) + /// Set rapidity + void setY(float y_min, float y_max) { - mEtaMin = eta_min; - mEtaMax = eta_max; + mYmin = y_min; + mYmax = y_max; } /// Set pseudorapidity @@ -131,12 +131,13 @@ public: for (int i = 0; i < mNinjected[mInjectionIndex]; ++i) { const double pt = gRandom->Uniform(mPtMin[mInjectionIndex], mPtMax[mInjectionIndex]); - const double eta = gRandom->Uniform(mEtaMin, mEtaMax); + const double rapidity = gRandom->Uniform(mYmin, mYmax); const double phi = gRandom->Uniform(0, TMath::TwoPi()); const double px{pt * std::cos(phi)}; const double py{pt * std::sin(phi)}; - const double pz{pt * std::sinh(eta)}; - const double et{std::hypot(std::hypot(pt, pz), currentMass)}; + const double mt{std::hypot(pt, currentMass)}; + const double pz{mt * std::sinh(rapidity)}; + const double et{mt * std::cosh(rapidity)}; sign *= 1 - 2 * mAlternatingPDGsign; mParticles.push_back(TParticle(sign * currentPdg, 1, -1, -1, -1, -1, px, py, pz, et, 0., 0., 0., 0.)); // make sure status code is encoded properly. Transport flag will be set by default and we have nothing @@ -155,8 +156,8 @@ private: std::vector mPtMin; /// minimum transverse momentum for generated particles std::vector mPtMax; /// maximum transverse momentum for generated particles - double mEtaMin = -1.; /// minimum pseudorapidity for generated particles - double mEtaMax = +1.; /// maximum pseudorapidity for generated particles + double mYmin = -1.; /// minimum rapidity for generated particles + double mYmax = +1.; /// maximum rapidity for generated particles bool mAlternatingPDGsign = true; /// bool to randomize the PDG code of the core particle From fc2fb6ca0c55b0cd7e9f93c10e13bbb5f4bdb885 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 4 Aug 2023 16:43:47 +0200 Subject: [PATCH 0012/1239] Workflow parser: Take SHM size from what AliECS provides, support larger SHMSIZE and process multiplicity for MI100 node --- DATA/production/production.desc | 4 ++-- DATA/tools/epn/gen_topo_o2dpg.sh | 2 ++ DATA/tools/parse | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/DATA/production/production.desc b/DATA/production/production.desc index a64d0eb9a..887645e8a 100644 --- a/DATA/production/production.desc +++ b/DATA/production/production.desc @@ -1,3 +1,3 @@ -synchronous-workflow: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 SHMSIZE=120259084288 production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 SHMSIZE=120259084288 production/dpl-workflow.sh" +synchronous-workflow: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 production/dpl-workflow.sh" synchronous-workflow-1numa: "O2PDPSuite" reco,128,126,"SYNCMODE=1 production/dpl-workflow.sh" -synchronous-workflow-calib: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 SHMSIZE=120259084288 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 SHMSIZE=120259084288 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_TF SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_SPORADIC SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,128,"AGGREGATOR_TASKS=TPC_IDCBOTH_SAC SHMSIZE=137438953472 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" +synchronous-workflow-calib: "O2PDPSuite" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=0 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" reco,128,126,"GPU_NUM_MEM_REG_CALLBACKS=5 SYNCMODE=1 NUMAGPUIDS=1 NUMAID=1 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/dpl-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_TF SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,32,"AGGREGATOR_TASKS=BARREL_SPORADIC SHMSIZE=68719476736 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=CALO_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=MUON_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_TF SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,16,"AGGREGATOR_TASKS=FORWARD_SPORADIC SHMSIZE=34359738368 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" calib,128,"AGGREGATOR_TASKS=TPC_IDCBOTH_SAC SHMSIZE=137438953472 WORKFLOW_PARAMETERS+=,CALIB_PROXIES production/aggregator-workflow.sh" diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 7f6345295..c930245f9 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -18,6 +18,8 @@ if [[ -z $DDMODE ]] && [[ -z $DDWORKFLOW ]]; then echo Either \$DDMODE or \$DDWO if [[ -z "$MULTIPLICITY_FACTOR_RAWDECODERS" ]]; then echo \$MULTIPLICITY_FACTOR_RAWDECODERS missing; exit 1; fi # Process multiplicity scaling parameter if [[ -z "$MULTIPLICITY_FACTOR_CTFENCODERS" ]]; then echo \$MULTIPLICITY_FACTOR_CTFENCODERS missing; exit 1; fi # Process multiplicity scaling parameter if [[ -z "$MULTIPLICITY_FACTOR_REST" ]]; then echo \$MULTIPLICITY_FACTOR_REST missing; exit 1; fi # Process multiplicity scaling parameter +if [[ -z "$RECOSHMSIZE" ]]; then echo \$RECOSHMSIZE missing; exit 1; fi # SHM Size for reconstruction collections +if [[ -z "$DDSHMSIZE" ]]; then echo \$DDSHMSIZE missing; exit 1; fi # SHM Size for DD # Check settings coming from the EPN if [[ -z "$FILEWORKDIR" ]]; then echo \$FILEWORKDIR missing; exit 1; fi diff --git a/DATA/tools/parse b/DATA/tools/parse index c1c5dd207..e7d82d2a8 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -29,6 +29,10 @@ if not 'DDWORKFLOW' in os.environ and not 'DDMODE' in os.environ: print('Need either $DDWORKFLOW or $DDMODE env variable') exit(1) +if not 'RECOSHMSIZE' in os.environ or not 'DDSHMSIZE' in os.environ: + print('RECOSHMSIZE or DDSHMSIZE setting missing') + exit(1) + NO_PROCESSING_MODE=0 if not 'DDWORKFLOW' in os.environ: os.environ['DDWORKFLOW'] = 'tools/datadistribution_workflows/dd-' + os.environ['DDMODE'] + '.xml' @@ -111,7 +115,7 @@ for line in f: print('Cannot use DPL workflow together with DD mode', os.environ['DDMODE']) raise - def processSubtopology(i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK): + def processSubtopology(i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK, mi100node): filename = tmpdir + '/wf' + str(i) + '.dds' calibcores = '1' if args[i].startswith('reco'): @@ -130,16 +134,20 @@ for line in f: print('Adding', wf[0], 'workflow (', wf[2], '-', wf[1], 'nodes):', wf[3]) reconodes = max(reconodes, int(wf[1])) reconodesmin = max(reconodesmin, int(wf[2])) - if reco_num_nodes_override == 0: - os.environ['RECO_NUM_NODES_WORKFLOW'] = wf[1] command_preopt = 'GEN_TOPO_CALIB_WORKFLOW=' + str(is_calib_workflow) + if not is_calib_workflow: + command_preopt += ' SHMSIZE=' + (str(int(int(os.environ['RECOSHMSIZE']) * 3 / 2)) if mi100node else os.environ['RECOSHMSIZE']) + if mi100node: + command_preopt += ' EPN_GLOBAL_SCALING="3/2"' + if reco_num_nodes_override == 0: + command_preopt += ' RECO_NUM_NODES_WORKFLOW=' + wf[1] if is_calib_workflow: command_preopt += ' GEN_TOPO_CALIB_NCORES=' + calibcores if os.environ['WORKFLOWMODE'] == 'dds': command_preopt += ' WORKFLOWMODE_FILE=' + filename command = command_preopt + ' GLOBALDPLOPT+=" -b --dds-workflow-suffix _' + wf[0] + str(i) + '" ' + wf[3] if os.environ['WORKFLOWMODE'] == 'print': - command += ' > ' + filename + command += ' > ' + filename print('Running DPL command', command) retVal = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) tmpchk = retVal.stderr.decode() + retVal.stdout.decode() @@ -175,7 +183,7 @@ for line in f: subtopologyThreads = [] threadsOK = [0] * len(args) for i in range(2, len(args)): - t = threading.Thread(target = processSubtopology, args = (i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK)) + t = threading.Thread(target = processSubtopology, args = (i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK, False)) t.start() if 'GEN_TOPO_SINGLE_THREAD' in os.environ and int(os.environ['GEN_TOPO_SINGLE_THREAD']): t.join() From 9f22750ff7c481d501f286b98530f305957d7e94 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 4 Aug 2023 16:56:45 +0200 Subject: [PATCH 0013/1239] gen_topo.sh wrapper: Allow setting odc-epn-topo command externally via env variable --- DATA/tools/epn/gen_topo.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 0070b4598..5ffc64303 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -39,8 +39,10 @@ else # If there is already something of ODC or O2PDPSuite in the environment, we should remove it to avoid collisions. # We set the odc-epn-topo command to be used explicitly though. # Note this happens only in case of on the fly generation when we run online, in case of tests this is not needed. - export GEN_TOPO_ODC_EPN_TOPO_CMD=`which odc-epn-topo` - [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; } + if [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]]; then + export GEN_TOPO_ODC_EPN_TOPO_CMD=`which odc-epn-topo` + [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; } + fi module purge &> /dev/null fi fi From d8105c3c659259de3c80dbb7a5bc9de16cb2fe02 Mon Sep 17 00:00:00 2001 From: Ruben Shahoyan Date: Fri, 4 Aug 2023 17:34:32 +0200 Subject: [PATCH 0014/1239] StrangenessTracking called from the SVertexer following O2 PR11713 (#1161) Co-authored-by: shahoian --- MC/bin/o2dpg_sim_workflow.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 01c347281..ce62f4c13 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1273,28 +1273,24 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # Take None as default, we only add more if nothing from anchorConfig svfinder_sources = anchorConfig.get("o2-secondary-vertexing-workflow-options",{}).get("vertexing-sources", None) if not svfinder_sources: - svfinder_sources = "ITS,ITS-TPC,TPC-TRD,TPC-TOF,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" - if isActive("MID"): - svfinder_sources += ",MID" + svfinder_sources = "ITS,ITS-TPC,TPC-TRD,TPC-TOF,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" + if isActive("MID"): + svfinder_sources += ",MID" SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] + # strangeness tracking is now called from the secondary vertexer + if args.no_strangeness_tracking: + SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' + # if enabled, it may require MC labels + else: + SVFINDERtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(SVFINDERtask) - #strangeness tracking - if not args.no_strangeness_tracking: - STRACKINGtask = createTask(name='stracking_'+str(tf), needs=[SVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='5000') - STRACKINGtask['cmd'] = '${O2_ROOT}/bin/o2-strangeness-tracking-workflow ' - STRACKINGtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] - STRACKINGtask['cmd'] += getDPL_global_options(bigshm=True) + putConfigValuesNew(['strtracker'], {"NameConf.mDirMatLUT" : ".."}) - workflow['stages'].append(STRACKINGtask) - # ----------- # produce AOD # ----------- # TODO This needs further refinement, sources and dependencies should be constructed dynamically aodinfosources = 'ITS,MFT,MCH,TPC,ITS-TPC,MFT-MCH,ITS-TPC-TOF,TPC-TOF,FT0,FDD,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF' aodneeds = [PVFINDERtask['name'], SVFINDERtask['name']] - if not args.no_strangeness_tracking: - aodneeds += [ STRACKINGtask['name'] ] if isActive('CTP'): aodinfosources += ',CTP' From 573b8150098df0c775c143189cd25c3014d060bc Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Wed, 9 Aug 2023 14:39:28 +0200 Subject: [PATCH 0015/1239] Circumvent memory corruption when downloading from CCDB --- MC/bin/o2dpg_sim_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index ce62f4c13..1844b062e 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -503,7 +503,8 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # reco. This is necessary to have consistency (decalibration and calibration) between digitization and reconstruction ... until digitization can # also apply this effect via CCDB. TPC_SPACECHARGE_DOWNLOADER_TASK = createTask(name='tpc_spacecharge_downloader', cpu='0') -TPC_SPACECHARGE_DOWNLOADER_TASK['cmd'] = '[ "${O2DPG_ENABLE_TPC_DISTORTIONS}" ] || ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch -p TPC/Calib/CorrectionMapRef TPC/Calib/CorrectionMap --timestamp 1 --created-not-after ' + str(args.condition_not_after) + ' -d ${ALICEO2_CCDB_LOCALCACHE}' +TPC_SPACECHARGE_DOWNLOADER_TASK['cmd'] = '[ "${O2DPG_ENABLE_TPC_DISTORTIONS}" ] || { ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch -p TPC/Calib/CorrectionMapRef --timestamp 1 --created-not-after ' + str(args.condition_not_after) + ' -d ${ALICEO2_CCDB_LOCALCACHE} ; ' \ + '${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch -p TPC/Calib/CorrectionMap --timestamp 1 --created-not-after ' + str(args.condition_not_after) + ' -d ${ALICEO2_CCDB_LOCALCACHE} ; }' workflow['stages'].append(TPC_SPACECHARGE_DOWNLOADER_TASK) From 94b3b970ed26aa9a5663dba81030d218a3d5db3a Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Wed, 9 Aug 2023 09:58:29 +0200 Subject: [PATCH 0016/1239] Correct CTP offset of 3BCs observed in LHC23zd-zt --- .../configurations/2022/LHC22f/apass1/setenv_extra.sh | 3 +++ DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh index 3399c86c9..55f5561c1 100644 --- a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh @@ -72,6 +72,9 @@ fi CTP_BC_SHIFT=0 if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then CTP_BC_SHIFT=-294 +if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539908 ]]; then + # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zt + CTP_BC_SHIFT=-3 fi if [[ $PERIOD == "LHC22s" ]]; then # CTP asked to extract their digits diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 0db47ea11..fa099c8bc 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -82,6 +82,10 @@ CTP_BC_SHIFT=0 if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then CTP_BC_SHIFT=-294 fi +if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539908 ]]; then + # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zt + CTP_BC_SHIFT=-3 +fi if [[ $PERIOD == "LHC22s" ]]; then # CTP asked to extract their digits add_comma_separated ADD_EXTRA_WORKFLOW "o2-ctp-digit-writer" From 7517d40c4b1b811b72652322651d3154674a11a3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 10 Aug 2023 21:37:25 +0200 Subject: [PATCH 0017/1239] parser: Add option to generate topologies for MI100 nodes --- DATA/tools/parse | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index e7d82d2a8..f77df9aff 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -84,6 +84,7 @@ for line in f: reconodes = 0 reconodesmin = 0 recoworkflows = [] + recoworkflows_mi100 = [] calibworkflows = [] calibworkflowsdds = [] print('Found topology', sys.argv[2], '-', args) @@ -115,8 +116,8 @@ for line in f: print('Cannot use DPL workflow together with DD mode', os.environ['DDMODE']) raise - def processSubtopology(i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK, mi100node): - filename = tmpdir + '/wf' + str(i) + '.dds' + def processSubtopology(i, wfCount, args, tmpdir, reconodes, reconodesmin, recoworkflows, recoworkflows_mi100, calibworkflows, calibworkflowsdds, threadsOK, mi100node): + filename = tmpdir + '/wf' + str(wfCount) + '.dds' calibcores = '1' if args[i].startswith('reco'): wf = args[i].split(',', 3) @@ -138,14 +139,14 @@ for line in f: if not is_calib_workflow: command_preopt += ' SHMSIZE=' + (str(int(int(os.environ['RECOSHMSIZE']) * 3 / 2)) if mi100node else os.environ['RECOSHMSIZE']) if mi100node: - command_preopt += ' EPN_GLOBAL_SCALING="3/2"' + command_preopt += ' EPN_NODE_MI100=1 EPN_GLOBAL_SCALING="3/2"' if reco_num_nodes_override == 0: command_preopt += ' RECO_NUM_NODES_WORKFLOW=' + wf[1] if is_calib_workflow: command_preopt += ' GEN_TOPO_CALIB_NCORES=' + calibcores if os.environ['WORKFLOWMODE'] == 'dds': command_preopt += ' WORKFLOWMODE_FILE=' + filename - command = command_preopt + ' GLOBALDPLOPT+=" -b --dds-workflow-suffix _' + wf[0] + str(i) + '" ' + wf[3] + command = command_preopt + ' GLOBALDPLOPT+=" -b --dds-workflow-suffix _' + wf[0] + str(wfCount) + '" ' + wf[3] if os.environ['WORKFLOWMODE'] == 'print': command += ' > ' + filename print('Running DPL command', command) @@ -163,9 +164,12 @@ for line in f: print('stderr:', retVal.stderr.decode()) print('stdout:', retVal.stdout.decode()) raise - threadsOK[i] = 1 + threadsOK[wfCount] = 1 if args[i].startswith('reco'): - recoworkflows.append(filename) + if mi100node: + recoworkflows_mi100.append(filename) + else: + recoworkflows.append(filename) elif args[i].startswith('calib'): if os.environ['WORKFLOWMODE'] == 'dds': isempty = 1 @@ -181,18 +185,24 @@ for line in f: calibworkflowsdds.append(filename + ':' + calibcores) subtopologyThreads = [] - threadsOK = [0] * len(args) + threadsOK = [] + wfCount = 0 for i in range(2, len(args)): - t = threading.Thread(target = processSubtopology, args = (i, args, tmpdir, reconodes, reconodesmin, recoworkflows, calibworkflows, calibworkflowsdds, threadsOK, False)) - t.start() - if 'GEN_TOPO_SINGLE_THREAD' in os.environ and int(os.environ['GEN_TOPO_SINGLE_THREAD']): - t.join() - if not threadsOK[i]: - raise - subtopologyThreads.append(t) + for nodetype in range(0, 2): + if nodetype == 1 and (not args[i].startswith('reco') or not 'GEN_TOPO_MI100_NODES' in os.environ or os.environ['GEN_TOPO_MI100_NODES'] == 0): + continue + threadsOK.append(0) + t = threading.Thread(target = processSubtopology, args = (i, wfCount, args, tmpdir, reconodes, reconodesmin, recoworkflows, recoworkflows_mi100, calibworkflows, calibworkflowsdds, threadsOK, nodetype)) + t.start() + if 'GEN_TOPO_SINGLE_THREAD' in os.environ and int(os.environ['GEN_TOPO_SINGLE_THREAD']): + t.join() + if not threadsOK[wfCount]: + raise + subtopologyThreads.append(t) + wfCount += 1 for t in subtopologyThreads: t.join() - for i in range(2, len(args)): + for i in range(0, wfCount): if not threadsOK[i]: raise if reco_num_nodes_override > 0: @@ -223,6 +233,8 @@ for line in f: odccommand += ' --dd ' + filename if len(recoworkflows): odccommand += ' --reco ' + ' '.join(recoworkflows) + if len(recoworkflows_mi100): + odccommand += ' --reco100 ' + ' '.join(recoworkflows_mi100) odccommand += ' --n ' + str(reconodes) odccommand += ' --nmin ' + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin if len(calibworkflows): @@ -261,6 +273,8 @@ for line in f: outf = open(sys.argv[3], 'w+') for i in recoworkflows: outf.write('# RECO workflow\n\n' + open(i, 'r').read() + '\n\n') + for i in recoworkflows_mi100: + outf.write('# RECO MI100 workflow\n\n' + open(i, 'r').read() + '\n\n') for i in calibworkflows: outf.write('# CALIB workflow\n\n' + open(i, 'r').read() + '\n\n') print('Done') From f3575990003e966c60e85ae4e1c04a74b62a0ea2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 10 Aug 2023 21:52:05 +0200 Subject: [PATCH 0018/1239] parser: Add timing information for workflow generation and topology merging commands --- DATA/tools/parse | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DATA/tools/parse b/DATA/tools/parse index f77df9aff..42fa4b71b 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -8,6 +8,7 @@ import tempfile import re import datetime import random +import time if not os.path.exists('tools/parse'): print('Running from incorrect directory') @@ -150,7 +151,9 @@ for line in f: if os.environ['WORKFLOWMODE'] == 'print': command += ' > ' + filename print('Running DPL command', command) + starttime = time.time() retVal = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + print('Execution time: ', time.time() - starttime) tmpchk = retVal.stderr.decode() + retVal.stdout.decode() haserror = 0 if tmpchk.find('[FATAL]') != -1 or tmpchk.find('[ERROR]') != -1 or tmpchk.find(': command not found') != -1 or tmpchk.find('No such file or directory') != -1 or tmpchk.find('bash: syntax error') != -1: @@ -251,9 +254,11 @@ for line in f: odccommand += '"' odccommand += ' -o ' + sys.argv[3] print('Running ODC command', odccommand) + starttime = time.time() if os.system(odccommand) != 0: print('\nError running odc: ', odccommand) raise + print('Execution time: ', time.time() - starttime) if os.path.exists(os.environ['GEN_TOPO_QC_JSON_FILE']): command = 'sed -i \'s/&/\&/g; s//\>/g; s/"/\"/g; s/\'"\'"\'/\'/g\' ' + os.environ['GEN_TOPO_QC_JSON_FILE'] print('Running SED command for DPL JSON XML Escaping', command) From 07c5d3c0d4b465034946aba22790917422149305 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 11 Aug 2023 10:59:08 +0200 Subject: [PATCH 0019/1239] gen_topo.sh: We can have different zones for produciton / staging, add MI100 zones for staging --- DATA/tools/epn/gen_topo.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 5ffc64303..cb107d5ae 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -23,7 +23,13 @@ else [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=$HOME/gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. mkdir -p $HOME/gen_topo fi -[[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]] && export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone online --calibzone calib" # Arguments to pass to odc-epn-topo command +if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then + if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" + else + export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone online --calibzone calib" + fi +fi [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://127.0.0.1:8084" # CCDB server to use if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi From 0907a335a035e503604137d90e00377effa27e39 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 11 Aug 2023 11:00:56 +0200 Subject: [PATCH 0020/1239] Parser: add qc.json to all , before it was bogus and depended on the order of statements, and could fail randomly --- DATA/tools/parse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index 42fa4b71b..0a7bdd770 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -266,7 +266,7 @@ for line in f: print('Error running sed on JSON file') command = 'sed -i ' + \ '-e \'s,' + os.environ['GEN_TOPO_QC_JSON_FILE'] + ',${DDS_LOCATION}/qc.json.asset,\' ' + \ - '-e \'/^ *.*o2-qc --id.*qc.json.asset.*<\/exe>$/a qc.json\' ' + \ + '-e \'/^ *$/a qc.json\' ' + \ '-e \'/^ *$/a \' ' + \ From 966f5e41b0e949312d5d28015c90cb028b9f56ae Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 11 Aug 2023 11:28:36 +0200 Subject: [PATCH 0021/1239] gen_topo_o2dpg.sh: Make fetching cached topology go through the same code path as full gen topo generation --- DATA/tools/epn/gen_topo_o2dpg.sh | 89 +++++++++++++++++--------------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index c930245f9..223c7e037 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -55,53 +55,56 @@ if [[ "0$DDMODE" == "0discard" ]] || [[ "0$DDMODE" == "0disk" ]]; then fi mkdir -p $GEN_TOPO_WORKDIR/cache || { echo Error creating directory 1>&2; exit 1; } -if [[ $GEN_TOPO_HASH == 1 ]]; then - cd $GEN_TOPO_WORKDIR || { echo Cannot enter work dir 1>&2; exit 1; } - if [[ ! -d O2DPG ]]; then git clone https://github.com/AliceO2Group/O2DPG.git 1>&2 || { echo O2DPG checkout failed 1>&2; exit 1; }; fi - if [[ "0$GEN_TOPO_ONTHEFLY" == "01" && ! -z $GEN_TOPO_CACHE_HASH ]]; then - export GEN_TOPO_CACHEABLE=1 - fi - if [[ "0$GEN_TOPO_CACHEABLE" == "01" && -f cache/$GEN_TOPO_CACHE_HASH ]]; then - if [[ "0$GEN_TOPO_WIPE_CACHE" == "01" ]]; then - rm -f cache/$GEN_TOPO_CACHE_HASH +while true; do + if [[ $GEN_TOPO_HASH == 1 ]]; then + cd $GEN_TOPO_WORKDIR || { echo Cannot enter work dir 1>&2; exit 1; } + if [[ ! -d O2DPG ]]; then git clone https://github.com/AliceO2Group/O2DPG.git 1>&2 || { echo O2DPG checkout failed 1>&2; exit 1; }; fi + if [[ "0$GEN_TOPO_ONTHEFLY" == "01" && ! -z $GEN_TOPO_CACHE_HASH ]]; then + export GEN_TOPO_CACHEABLE=1 fi - echo Reusing cached XML topology 1>&2 - touch cache/$GEN_TOPO_CACHE_HASH - cat cache/$GEN_TOPO_CACHE_HASH - exit 0 - fi - cd O2DPG - git checkout $GEN_TOPO_SOURCE &> /dev/null - if [[ $? != 0 ]]; then - git fetch --tags origin 1>&2 || { echo Repository update failed 1>&2; exit 1; } - git checkout $GEN_TOPO_SOURCE &> /dev/null || { echo commit does not exist 1>&2; exit 1; } - fi - # At a tag, or a detached non-dirty commit, but not on a branch - if ! git describe --exact-match --tags HEAD &> /dev/null && ( git symbolic-ref -q HEAD &> /dev/null || ! git diff-index --quiet HEAD &> /dev/null ); then - unset GEN_TOPO_CACHEABLE + if [[ "0$GEN_TOPO_CACHEABLE" == "01" && -f cache/$GEN_TOPO_CACHE_HASH ]]; then + if [[ "0$GEN_TOPO_WIPE_CACHE" == "01" ]]; then + rm -f cache/$GEN_TOPO_CACHE_HASH + fi + echo Reusing cached XML topology 1>&2 + touch cache/$GEN_TOPO_CACHE_HASH + cp cache/$GEN_TOPO_CACHE_HASH $GEN_TOPO_WORKDIR/output.xml + break + fi + cd O2DPG + git checkout $GEN_TOPO_SOURCE &> /dev/null + if [[ $? != 0 ]]; then + git fetch --tags origin 1>&2 || { echo Repository update failed 1>&2; exit 1; } + git checkout $GEN_TOPO_SOURCE &> /dev/null || { echo commit does not exist 1>&2; exit 1; } + fi + # At a tag, or a detached non-dirty commit, but not on a branch + if ! git describe --exact-match --tags HEAD &> /dev/null && ( git symbolic-ref -q HEAD &> /dev/null || ! git diff-index --quiet HEAD &> /dev/null ); then + unset GEN_TOPO_CACHEABLE + fi + cd DATA + else + cd $GEN_TOPO_SOURCE || { echo Directory missing 1>&2; exit 1; } fi - cd DATA -else - cd $GEN_TOPO_SOURCE || { echo Directory missing 1>&2; exit 1; } -fi -export EPNSYNCMODE=1 -export O2DPG_ROOT=`realpath \`pwd\`/../` -echo Running topology generation to temporary file $GEN_TOPO_WORKDIR/output.xml 1>&2 -# Run stage 3 of GenTopo, now from the O2DPG version specified by the user -./tools/parse "$GEN_TOPO_LIBRARY_FILE" $GEN_TOPO_WORKFLOW_NAME $GEN_TOPO_WORKDIR/output.xml 1>&2 || { echo Error during workflow description parsing 1>&2; exit 1; } -if [[ "0$GEN_TOPO_CACHEABLE" == "01" ]]; then - cd $GEN_TOPO_WORKDIR - if [[ `ls cache/ | wc -l` -ge 1000 ]]; then - ls -t cache/* | tail -n +1000 | xargs rm + export EPNSYNCMODE=1 + export O2DPG_ROOT=`realpath \`pwd\`/../` + echo Running topology generation to temporary file $GEN_TOPO_WORKDIR/output.xml 1>&2 + # Run stage 3 of GenTopo, now from the O2DPG version specified by the user + ./tools/parse "$GEN_TOPO_LIBRARY_FILE" $GEN_TOPO_WORKFLOW_NAME $GEN_TOPO_WORKDIR/output.xml 1>&2 || { echo Error during workflow description parsing 1>&2; exit 1; } + if [[ "0$GEN_TOPO_CACHEABLE" == "01" ]]; then + cd $GEN_TOPO_WORKDIR + if [[ `ls cache/ | wc -l` -ge 1000 ]]; then + ls -t cache/* | tail -n +1000 | xargs rm + fi + cp $GEN_TOPO_WORKDIR/output.xml cache/$GEN_TOPO_CACHE_HASH fi - cp $GEN_TOPO_WORKDIR/output.xml cache/$GEN_TOPO_CACHE_HASH -fi -if [[ ! -z $ECS_ENVIRONMENT_ID && -d "/var/log/topology/" && $USER == "epn" ]]; then - GEN_TOPO_LOG_FILE=/var/log/topology/topology-$(date -u +%Y%m%d-%H%M%S)-$ECS_ENVIRONMENT_ID.xml - cp $GEN_TOPO_WORKDIR/output.xml $GEN_TOPO_LOG_FILE - nohup gzip $GEN_TOPO_LOG_FILE &> /dev/null & -fi + if [[ ! -z $ECS_ENVIRONMENT_ID && -d "/var/log/topology/" && $USER == "epn" ]]; then + GEN_TOPO_LOG_FILE=/var/log/topology/topology-$(date -u +%Y%m%d-%H%M%S)-$ECS_ENVIRONMENT_ID.xml + cp $GEN_TOPO_WORKDIR/output.xml $GEN_TOPO_LOG_FILE + nohup gzip $GEN_TOPO_LOG_FILE &> /dev/null & + fi + break +done cat $GEN_TOPO_WORKDIR/output.xml echo Removing temporary output file $GEN_TOPO_WORKDIR/output.xml 1>&2 From 77458422cc9d16d2ad63e3c42ac6af0ba505b2e2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 11 Aug 2023 13:33:11 +0200 Subject: [PATCH 0022/1239] parser: pass MI100 number of nodes option to topology merger --- DATA/tools/parse | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DATA/tools/parse b/DATA/tools/parse index 0a7bdd770..28298cdc5 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -240,6 +240,8 @@ for line in f: odccommand += ' --reco100 ' + ' '.join(recoworkflows_mi100) odccommand += ' --n ' + str(reconodes) odccommand += ' --nmin ' + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin + if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) != -1: + odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin) if len(calibworkflows): calibworkflowsdds.sort(key=lambda x:int(x.split(':')[1])*-1) odccommand += ' --calib ' + ' '.join(calibworkflowsdds) From 133407eec9d5d9507667b93ab4c62f1d4cbae3c5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 11 Aug 2023 14:17:04 +0200 Subject: [PATCH 0023/1239] parser: --n100 0 is not supported --- DATA/tools/parse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index 28298cdc5..0fd82a042 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -240,7 +240,7 @@ for line in f: odccommand += ' --reco100 ' + ' '.join(recoworkflows_mi100) odccommand += ' --n ' + str(reconodes) odccommand += ' --nmin ' + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin - if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) != -1: + if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) > 0: odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin) if len(calibworkflows): calibworkflowsdds.sort(key=lambda x:int(x.split(':')[1])*-1) From d35acf9567f70f8174938552e3c7d26714a7f1bf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 14 Aug 2023 10:49:41 +0200 Subject: [PATCH 0024/1239] parser: correctly compute minimum number of MI100 nodes --- DATA/tools/parse | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index 0fd82a042..45f08b943 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -241,7 +241,11 @@ for line in f: odccommand += ' --n ' + str(reconodes) odccommand += ' --nmin ' + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) > 0: - odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin) + if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != '': + reconodesmin100 = max(1, int(os.environ['GEN_TOPO_MI100_NODES']) - int(os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'])) + else + reconodesmin100 = min(int(os.environ['GEN_TOPO_MI100_NODES']), reconodesmin) + odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin100) if len(calibworkflows): calibworkflowsdds.sort(key=lambda x:int(x.split(':')[1])*-1) odccommand += ' --calib ' + ' '.join(calibworkflowsdds) From d7486fdaa8b5329380dc65fec8f2af26857ed2a1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 14 Aug 2023 14:08:05 +0200 Subject: [PATCH 0025/1239] Fix Typo --- DATA/tools/parse | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index 45f08b943..101b70c28 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -243,7 +243,7 @@ for line in f: if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) > 0: if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != '': reconodesmin100 = max(1, int(os.environ['GEN_TOPO_MI100_NODES']) - int(os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'])) - else + else: reconodesmin100 = min(int(os.environ['GEN_TOPO_MI100_NODES']), reconodesmin) odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin100) if len(calibworkflows): From cc71545f5e13d5f5bcb644598fb96c65282dbc73 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Mon, 14 Aug 2023 18:01:52 +0200 Subject: [PATCH 0026/1239] Change of laser workflows for Dense packade format (#1173) --- DATA/production/calib/tpc-laser-filter.sh | 8 ++++++-- DATA/production/calib/tpc-laser.sh | 8 +++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 63daeacab..7ffe4a446 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -52,18 +52,22 @@ PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSRE #echo GPU_CONFIG $GPU_CONFIG_KEYS; +LASER_DECODER_ADD='' + +if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then + LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" +fi o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ + | o2-tpc-raw-to-digits-workflow $ARGS_ALL ${LASER_DECODER_ADD} \ --input-spec "$CALIB_INSPEC" \ --configKeyValues "TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \ --pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root \ --pipeline tpc-raw-to-digits-0:20 \ --remove-duplicates \ --send-ce-digits \ - --decoder-type 0 \ | o2-tpc-reco-workflow $ARGS_ALL \ --input-type digitizer \ --output-type "tracks,disable-writer" \ diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index e48f1784d..a05854df3 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -78,7 +78,11 @@ if [[ ! -z ${TPC_CALIB_LANES_PAD_RAW:-} ]]; then fi +LASER_DECODER_ADD='' +if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then + LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" +fi @@ -86,14 +90,12 @@ fi o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ + | o2-tpc-raw-to-digits-workflow $ARGS_ALL ${LASER_DECODER_ADD}\ --input-spec "$CALIB_INSPEC" \ --configKeyValues "TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \ - --pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root \ --pipeline tpc-raw-to-digits-0:20 \ --remove-duplicates \ --send-ce-digits \ - --decoder-type 0 \ | o2-tpc-reco-workflow $ARGS_ALL \ --input-type digitizer \ --output-type "tracks,disable-writer" \ From dd353bf8cbe2c92ebe021515fa38e280f9349dce Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 15 Aug 2023 09:43:04 +0200 Subject: [PATCH 0027/1239] Add TRD t0 monitoring to calibrations (#1175) --- DATA/common/setenv_calib.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 800ba1c31..95cb13a9d 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -20,7 +20,7 @@ if has_detector_calib TPC && has_processing_step TPC_DEDX; then CAN_DO_CALIB_TPC if has_detector_calib TPC && has_detectors ITS TPC && has_detector_matching ITSTPC; then CAN_DO_CALIB_TPC_VDRIFTTGL=1; else CAN_DO_CALIB_TPC_VDRIFTTGL=0; fi if has_detector_calib TPC; then CAN_DO_CALIB_TPC_IDC=1; CAN_DO_CALIB_TPC_SAC=1; else CAN_DO_CALIB_TPC_IDC=0; CAN_DO_CALIB_TPC_SAC=0; fi if [[ ! -z ${FLP_IDS:-} && ! $FLP_IDS =~ (^|,)"145"(,|$) ]] || [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then CAN_DO_CALIB_TPC_SAC=0; fi -if has_detector_calib TRD && has_detectors ITS TPC TRD && has_detector_matching ITSTPCTRD; then CAN_DO_CALIB_TRD_VDRIFTEXB=1; CAN_DO_CALIB_TRD_GAIN=1; else CAN_DO_CALIB_TRD_VDRIFTEXB=0; CAN_DO_CALIB_TRD_GAIN=0; fi +if has_detector_calib TRD && has_detectors ITS TPC TRD && has_detector_matching ITSTPCTRD; then CAN_DO_CALIB_TRD_VDRIFTEXB=1; CAN_DO_CALIB_TRD_GAIN=1; CAN_DO_CALIB_TRD_T0=1; else CAN_DO_CALIB_TRD_VDRIFTEXB=0; CAN_DO_CALIB_TRD_GAIN=0; CAN_DO_CALIB_TRD_T0=0; fi if has_detector_calib EMC && has_detector_reco EMC; then CAN_DO_CALIB_EMC_BADCHANNELCALIB=1; CAN_DO_CALIB_EMC_TIMECALIB=1; else CAN_DO_CALIB_EMC_BADCHANNELCALIB=0; CAN_DO_CALIB_EMC_TIMECALIB=0; fi if has_detector_calib PHS && has_detector_reco PHS; then CAN_DO_CALIB_PHS_ENERGYCALIB=0; CAN_DO_CALIB_PHS_BADMAPCALIB=1; CAN_DO_CALIB_PHS_TURNONCALIB=1; CAN_DO_CALIB_PHS_RUNBYRUNCALIB=1; CAN_DO_CALIB_PHS_L1PHASE=1; else CAN_DO_CALIB_PHS_ENERGYCALIB=0; CAN_DO_CALIB_PHS_BADMAPCALIB=0; CAN_DO_CALIB_PHS_TURNONCALIB=0; CAN_DO_CALIB_PHS_RUNBYRUNCALIB=0; CAN_DO_CALIB_PHS_L1PHASE=0; fi if has_detector_calib CPV && has_detector_reco CPV; then CAN_DO_CALIB_CPV_GAIN=1; else CAN_DO_CALIB_CPV_GAIN=0; fi @@ -33,6 +33,7 @@ if has_detector_calib EMC && has_detector_reco EMC && [[ $SYNCMODE != 1 ]]; then # additional individual settings for calibration workflows if has_detector CTP; then export CALIB_TPC_SCDCALIB_CTP_INPUT="--enable-ctp"; else export CALIB_TPC_SCDCALIB_CTP_INPUT=""; fi +if [[ ${DISABLE_TRD_PH:-} == 1 ]]; then CAN_DO_CALIB_TRD_T0=0; fi # the slot length needs to be known both on the aggregator and the processing nodes, therefore it is defined (in seconds!) here : ${CALIB_TPC_SCDCALIB_SLOTLENGTH:=600} @@ -90,6 +91,9 @@ if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then if [[ $CAN_DO_CALIB_TRD_GAIN == 1 ]] ; then if [[ -z ${CALIB_TRD_GAIN+x} ]]; then CALIB_TRD_GAIN=1; fi fi + if [[ $CAN_DO_CALIB_TRD_T0 == 1 ]] ; then + if [[ -z ${CALIB_TRD_T0+x} ]]; then CALIB_TRD_T0=1; fi + fi # calibrations for EMC if [[ $CAN_DO_CALIB_EMC_BADCHANNELCALIB == 1 ]]; then @@ -158,6 +162,7 @@ fi ( [[ -z ${CALIB_TPC_VDRIFTTGL:-} ]] || [[ $CAN_DO_CALIB_TPC_VDRIFTTGL == 0 ]] ) && CALIB_TPC_VDRIFTTGL=0 ( [[ -z ${CALIB_TRD_VDRIFTEXB:-} ]] || [[ $CAN_DO_CALIB_TRD_VDRIFTEXB == 0 ]] ) && CALIB_TRD_VDRIFTEXB=0 ( [[ -z ${CALIB_TRD_GAIN:-} ]] || [[ $CAN_DO_CALIB_TRD_GAIN == 0 ]] ) && CALIB_TRD_GAIN=0 +( [[ -z ${CALIB_TRD_T0:-} ]] || [[ $CAN_DO_CALIB_TRD_T0 == 0 ]] ) && CALIB_TRD_T0=0 ( [[ -z ${CALIB_EMC_BADCHANNELCALIB:-} ]] || [[ $CAN_DO_CALIB_EMC_BADCHANNELCALIB == 0 ]] ) && CALIB_EMC_BADCHANNELCALIB=0 ( [[ -z ${CALIB_EMC_TIMECALIB:-} ]] || [[ $CAN_DO_CALIB_EMC_TIMECALIB == 0 ]] ) && CALIB_EMC_TIMECALIB=0 ( [[ -z ${CALIB_PHS_ENERGYCALIB:-} ]] || [[ $CAN_DO_CALIB_PHS_ENERGYCALIB == 0 ]] ) && CALIB_PHS_ENERGYCALIB=0 @@ -185,6 +190,7 @@ if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then echo "CALIB_PHS_L1PHASE = $CALIB_PHS_L1PHASE" 1>&2 echo "CALIB_TRD_VDRIFTEXB = $CALIB_TRD_VDRIFTEXB" 1>&2 echo "CALIB_TRD_GAIN = $CALIB_TRD_GAIN" 1>&2 + echo "CALIB_TRD_T0 = $CALIB_TRD_T0" 1>&2 echo "CALIB_TPC_TIMEGAIN = $CALIB_TPC_TIMEGAIN" 1>&2 echo "CALIB_TPC_RESPADGAIN = $CALIB_TPC_RESPADGAIN" 1>&2 echo "CALIB_TPC_IDC = $CALIB_TPC_IDC" 1>&2 @@ -223,6 +229,7 @@ if [[ -z ${CALIBDATASPEC_BARREL_TF:-} ]]; then # TRD if [[ $CALIB_TRD_VDRIFTEXB == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "angResHistoTRD:TRD/ANGRESHISTS/0"; fi if [[ $CALIB_TRD_GAIN == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "gainHistoTRD:TRD/GAINCALIBHISTS/0"; fi + if [[ $CALIB_TRD_T0 == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "trdph:TRD/PULSEHEIGHT/0"; fi fi # define spec for proxy for sporadic outputs from BARREL From f9de90821395aca459690d9fede4ccf3313d0873 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 14 Aug 2023 14:47:08 +0200 Subject: [PATCH 0028/1239] topology generation: fix some checks with quotes --- DATA/tools/epn/gen_topo.sh | 4 ++-- DATA/tools/epn/gen_topo_o2dpg.sh | 26 +++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index cb107d5ae..564753259 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -45,9 +45,9 @@ else # If there is already something of ODC or O2PDPSuite in the environment, we should remove it to avoid collisions. # We set the odc-epn-topo command to be used explicitly though. # Note this happens only in case of on the fly generation when we run online, in case of tests this is not needed. - if [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]]; then + if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then export GEN_TOPO_ODC_EPN_TOPO_CMD=`which odc-epn-topo` - [[ -z $GEN_TOPO_ODC_EPN_TOPO_CMD ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; } + [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; } fi module purge &> /dev/null fi diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 223c7e037..60f111aa5 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -5,16 +5,16 @@ [[ -z "$IS_SIMULATED_DATA" ]] && export IS_SIMULATED_DATA=0 # by default we are processing raw data # Check settings coming from AliECS via env variables -if [[ -z $GEN_TOPO_HASH ]]; then echo \$GEN_TOPO_HASH missing; exit 1; fi # Flag whether source is a hash or a folder -if [[ -z $GEN_TOPO_SOURCE ]]; then echo \$GEN_TOPO_SOURCE missing; exit 1; fi # O2DPG repository source, either a commit hash or a path -if [[ -z $GEN_TOPO_LIBRARY_FILE ]]; then echo \$GEN_TOPO_LIBRARY_FILE missing; exit 1; fi # Topology description library file in the DATA path of the O2DPG repository -if [[ -z $GEN_TOPO_WORKFLOW_NAME ]]; then echo \$GEN_TOPO_WORKFLOW_NAME missing; exit 1; fi # Workflow name in library file -if [[ -z ${WORKFLOW_DETECTORS+x} ]]; then echo \$WORKFLOW_DETECTORS missing; exit 1; fi # Comma-separated list of detectors to run processing for -if [[ -z ${WORKFLOW_DETECTORS_QC+x} && -z ${WORKFLOW_DETECTORS_EXCLUDE_QC+x} ]]; then echo \$WORKFLOW_DETECTORS_EXCLUDE_QC missing; exit 1; fi # Comma-separated list of detectors to run QC for -if [[ -z ${WORKFLOW_DETECTORS_CALIB+x} && -z ${WORKFLOW_DETECTORS_EXCLUDE_CALIB+x} ]]; then echo \$WORKFLOW_DETECTORS_EXCLUDE_CALIB missing; exit 1; fi # Comma-separated list of detectors to run calibration for -if [[ -z ${WORKFLOW_PARAMETERS+x} ]]; then echo \$WORKFLOW_PARAMETERS missing; exit 1; fi # Additional parameters for workflow -if [[ -z ${RECO_NUM_NODES_OVERRIDE+x} ]]; then echo \$RECO_NUM_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes -if [[ -z $DDMODE ]] && [[ -z $DDWORKFLOW ]]; then echo Either \$DDMODE or \$DDWORKFLOW must be set; exit 1; fi # Select data distribution workflow +if [[ -z "$GEN_TOPO_HASH" ]]; then echo \$GEN_TOPO_HASH missing; exit 1; fi # Flag whether source is a hash or a folder +if [[ -z "$GEN_TOPO_SOURCE" ]]; then echo \$GEN_TOPO_SOURCE missing; exit 1; fi # O2DPG repository source, either a commit hash or a path +if [[ -z "$GEN_TOPO_LIBRARY_FILE" ]]; then echo \$GEN_TOPO_LIBRARY_FILE missing; exit 1; fi # Topology description library file in the DATA path of the O2DPG repository +if [[ -z "$GEN_TOPO_WORKFLOW_NAME" ]]; then echo \$GEN_TOPO_WORKFLOW_NAME missing; exit 1; fi # Workflow name in library file +if [[ -z "${WORKFLOW_DETECTORS+x}" ]]; then echo \$WORKFLOW_DETECTORS missing; exit 1; fi # Comma-separated list of detectors to run processing for +if [[ -z "${WORKFLOW_DETECTORS_QC+x}" && -z "${WORKFLOW_DETECTORS_EXCLUDE_QC+x}" ]]; then echo \$WORKFLOW_DETECTORS_EXCLUDE_QC missing; exit 1; fi # Comma-separated list of detectors to run QC for +if [[ -z "${WORKFLOW_DETECTORS_CALIB+x}" && -z "${WORKFLOW_DETECTORS_EXCLUDE_CALIB+x}" ]]; then echo \$WORKFLOW_DETECTORS_EXCLUDE_CALIB missing; exit 1; fi # Comma-separated list of detectors to run calibration for +if [[ -z "${WORKFLOW_PARAMETERS+x}" ]]; then echo \$WORKFLOW_PARAMETERS missing; exit 1; fi # Additional parameters for workflow +if [[ -z "${RECO_NUM_NODES_OVERRIDE+x}" ]]; then echo \$RECO_NUM_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes +if [[ -z "$DDMODE" ]] && [[ -z "$DDWORKFLOW" ]]; then echo Either \$DDMODE or \$DDWORKFLOW must be set; exit 1; fi # Select data distribution workflow if [[ -z "$MULTIPLICITY_FACTOR_RAWDECODERS" ]]; then echo \$MULTIPLICITY_FACTOR_RAWDECODERS missing; exit 1; fi # Process multiplicity scaling parameter if [[ -z "$MULTIPLICITY_FACTOR_CTFENCODERS" ]]; then echo \$MULTIPLICITY_FACTOR_CTFENCODERS missing; exit 1; fi # Process multiplicity scaling parameter if [[ -z "$MULTIPLICITY_FACTOR_REST" ]]; then echo \$MULTIPLICITY_FACTOR_REST missing; exit 1; fi # Process multiplicity scaling parameter @@ -47,7 +47,7 @@ for i in `seq 1 100`; do GEN_TOPO_LOCKFILE=${GEN_TOPO_WORKDIR}/${i}.lock break done -[[ -z $GEN_TOPO_LOCKFILE ]] && { echo Topology generation could not obtained a work dir 1>&1; exit 1; } +[[ -z "$GEN_TOPO_LOCKFILE" ]] && { echo Topology generation could not obtained a work dir 1>&1; exit 1; } if [[ "0$DDMODE" == "0discard" ]] || [[ "0$DDMODE" == "0disk" ]]; then export GEN_TOPO_LIBRARY_FILE="production/no-processing.desc" @@ -59,7 +59,7 @@ while true; do if [[ $GEN_TOPO_HASH == 1 ]]; then cd $GEN_TOPO_WORKDIR || { echo Cannot enter work dir 1>&2; exit 1; } if [[ ! -d O2DPG ]]; then git clone https://github.com/AliceO2Group/O2DPG.git 1>&2 || { echo O2DPG checkout failed 1>&2; exit 1; }; fi - if [[ "0$GEN_TOPO_ONTHEFLY" == "01" && ! -z $GEN_TOPO_CACHE_HASH ]]; then + if [[ "0$GEN_TOPO_ONTHEFLY" == "01" && ! -z "$GEN_TOPO_CACHE_HASH" ]]; then export GEN_TOPO_CACHEABLE=1 fi if [[ "0$GEN_TOPO_CACHEABLE" == "01" && -f cache/$GEN_TOPO_CACHE_HASH ]]; then @@ -98,7 +98,7 @@ while true; do cp $GEN_TOPO_WORKDIR/output.xml cache/$GEN_TOPO_CACHE_HASH fi - if [[ ! -z $ECS_ENVIRONMENT_ID && -d "/var/log/topology/" && $USER == "epn" ]]; then + if [[ ! -z "$ECS_ENVIRONMENT_ID" && -d "/var/log/topology/" && $USER == "epn" ]]; then GEN_TOPO_LOG_FILE=/var/log/topology/topology-$(date -u +%Y%m%d-%H%M%S)-$ECS_ENVIRONMENT_ID.xml cp $GEN_TOPO_WORKDIR/output.xml $GEN_TOPO_LOG_FILE nohup gzip $GEN_TOPO_LOG_FILE &> /dev/null & From b125605095a67c509158a9b13b0010e74c66ffdc Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 15 Aug 2023 18:21:42 +0200 Subject: [PATCH 0029/1239] Update documentation --- DATA/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DATA/README.md b/DATA/README.md index 2fea52d9a..bc59da599 100644 --- a/DATA/README.md +++ b/DATA/README.md @@ -35,6 +35,14 @@ Another abstraction layer above the *workflows* are **topology descriptions**. T - DPL metrics and InfoLogger (not a requirement in the sense that something would fail, but without it is difficult to debug): - The workflow commands should contain `--monitoring-backend influxdb-unix:///tmp/telegraf.sock --resources-monitoring 60` for the DPL metrics. - `--infologger-severity $INFOLOGGER_SEVERITY` enables the infologger. +- Workflows should use as last command in the pipe the `o2-dpl-run` and add to it the `$GLOBALDPLOPT` env variable as command line argument. +- Workflows should source these files to get common functions: + ``` + source common/getCommonArgs.sh + source common/gen_topo_helper_functions.sh + ``` +- Workflows should aggregate their workflow parts in the `$WORKFLOW` variable, and use `add_W` to add a workflow, and `add_QC_from_consul` to add a QC workflow with a JSON file from consul. +- Calibration workflows must not pollute the production CCDB in the following case (`if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then` (bash script)), in this case please e.g. upload to `ccdb-test.cern.ch`. # Configuring and selecting workflow in AliECS: There are 3 ways foreseen to configure the *full topology* in AliECS: (currently only the manual XML option exists) From df9d9a211954d5c4c6cff7b9f1efa86a5827787f Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Wed, 16 Aug 2023 09:41:19 +0200 Subject: [PATCH 0030/1239] Workflow for pattern generator (#1174) --- .../production/calib/tpc-pattern-generator.sh | 38 +++++++++++++++++++ DATA/production/standalone-calibration.desc | 2 + 2 files changed, 40 insertions(+) create mode 100755 DATA/production/calib/tpc-pattern-generator.sh diff --git a/DATA/production/calib/tpc-pattern-generator.sh b/DATA/production/calib/tpc-pattern-generator.sh new file mode 100755 index 000000000..74bf683ce --- /dev/null +++ b/DATA/production/calib/tpc-pattern-generator.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +source common/setenv.sh +export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM +export GPUMEMSIZE=$(( 24 << 30 )) +export HOSTMEMSIZE=$(( 5 << 30 )) +export GPUTYPE="HIP" + +source common/getCommonArgs.sh + +if [ $NUMAGPUIDS != 0 ]; then + ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" +fi + +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" + +### Comment: MAKE SURE the channels match address=ipc://@tf-builder-pipe-0 +HOST=localhost +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + + + + + + + +o2-dpl-raw-proxy $ARGS_ALL \ + --dataspec "$PROXY_INSPEC" \ + --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ + | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ + --input-spec "$CALIB_INSPEC" \ + --configKeyValues "TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \ + --pipeline tpc-raw-to-digits-0:20 \ + --remove-duplicates \ + | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index 82f031586..e99c5c79e 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -34,6 +34,8 @@ TPC-laser: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE= TPC-laser-multi: "O2PDPSuite" reco,10,10,"SHMSIZE=$((64 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser-filter.sh" calib,1,"production/calib/tpc-laser-aggregator.sh" +TPC-pattern-generator: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-pattern-generator.sh" + MFT-noise-calibration: "O2PDPSuite" reco,20,20,"production/calib/mft-noise-processing.sh" calib,20,"production/calib/mft-noise-aggregator.sh" MCH-badchannel-calibration: "O2PDPSuite" reco,5,5,"production/calib/mch-badchannel-processing.sh" calib,20,"production/calib/mch-badchannel-aggregator.sh" From af7eecd149700787121fd30c956e6d251721420c Mon Sep 17 00:00:00 2001 From: iravasen Date: Wed, 16 Aug 2023 09:41:52 +0200 Subject: [PATCH 0031/1239] ITS: added calib workflow vresetd scan (#1149) * added new calib workflows for ITS * added workflow to process vresetd scan --- DATA/production/calib/its-threshold-processing.sh | 2 +- DATA/production/standalone-calibration.desc | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DATA/production/calib/its-threshold-processing.sh b/DATA/production/calib/its-threshold-processing.sh index de5b9fdd8..2214e2479 100755 --- a/DATA/production/calib/its-threshold-processing.sh +++ b/DATA/production/calib/its-threshold-processing.sh @@ -26,7 +26,7 @@ ADDITIONAL_OPTIONS_CAL="" if [ $RUNTYPE == "tuningbb" ]; then ADDITIONAL_OPTIONS_CAL="--min-vcasn 30 --max-vcasn 130" fi -if [ $RUNTYPE == "tot1row" ]; then +if [[ $RUNTYPE == "tot1row" || $RUNTYPE == "vresetd" ]]; then ADDITIONAL_OPTIONS_DEC="--allow-empty-rofs" ADDITIONAL_OPTIONS_CAL="--ninj 10" fi diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index e99c5c79e..885bad4e8 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -12,14 +12,16 @@ ITS-thr-short: "O2PDPSuite" reco,40,40,"RUNTYPE=thrshort production/calib/its-th ITS-thr-full: "O2PDPSuite" reco,80,80,"RUNTYPE=thrfull production/calib/its-threshold-processing.sh" calib,80,"RUNTYPE=thrfull production/calib/its-threshold-aggregator.sh" -ITS-thr-digital: "O2PDPSuite" reco,40,40,"RUNTYPE=digital production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=digital production/calib/its-threshold-aggregator.sh" +ITS-digital: "O2PDPSuite" reco,40,40,"RUNTYPE=digital production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=digital production/calib/its-threshold-aggregator.sh" -ITS-thr-pulselength: "O2PDPSuite" reco,40,40,"RUNTYPE=pulselength production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=pulselength production/calib/its-threshold-aggregator.sh" +ITS-pulselength: "O2PDPSuite" reco,40,40,"RUNTYPE=pulselength production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=pulselength production/calib/its-threshold-aggregator.sh" ITS-tot-1row: "O2PDPSuite" reco,40,40,"RUNTYPE=tot1row production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=tot1row production/calib/its-threshold-aggregator.sh" ITS-tot-fullfast: "O2PDPSuite" reco,40,40,"RUNTYPE=totfullfast production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=totfullfast production/calib/its-threshold-aggregator.sh" +ITS-vresetd: "O2PDPSuite" reco,5,5,"RUNTYPE=vresetd production/calib/its-threshold-processing.sh" calib,5,"RUNTYPE=vresetd production/calib/its-threshold-aggregator.sh" + TOF-diagnostic-calibration: "O2PDPSuite" reco,10,10,"SHMSIZE=120376524800 production/calib/tof-standalone-reco.sh" calib,4,"production/calib/tof-diagn-aggregator.sh" TOF-time-calibration: "O2PDPSuite" reco,10,10,"SHMSIZE=120376524800 production/calib/tof-standalone-cosmic-reco-time-calib.sh" calib,4,"production/calib/tof-time-calib-aggregator.sh" From 7b2f22150285b3a9e525ebe2ab5398cfc442e210 Mon Sep 17 00:00:00 2001 From: Piotr Konopka Date: Thu, 20 Jul 2023 13:48:47 +0200 Subject: [PATCH 0032/1239] Do not access consul directly from MCH bad channel wf download the file before instead --- DATA/production/calib/mch-badchannel-aggregator.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/calib/mch-badchannel-aggregator.sh b/DATA/production/calib/mch-badchannel-aggregator.sh index a582d3914..e362e373a 100755 --- a/DATA/production/calib/mch-badchannel-aggregator.sh +++ b/DATA/production/calib/mch-badchannel-aggregator.sh @@ -5,6 +5,7 @@ source common/setenv.sh # --------------------------------------------------------------------------------------------------------------------- # Set general arguments source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh PROXY_INSPEC="A:MCH/PDIGITS/0" CONSUL_ENDPOINT="alio2-cr1-hv-con01.cern.ch:8500" @@ -20,7 +21,7 @@ WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name mch-badchannel-input-proxy --d WORKFLOW+="o2-calibration-mch-badchannel-calib-workflow $ARGS_ALL --configKeyValues \"$BADCHANNEL_CONFIG\" | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ali-calib-dcs.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " -WORKFLOW+="o2-qc $ARGS_ALL --config consul-json://${CONSUL_ENDPOINT}/o2/components/qc/ANY/any/mch-badchannel | " +add_QC_from_consul "/o2/components/qc/ANY/any/mch-badchannel" "" WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" if [ $WORKFLOWMODE == "print" ]; then From 5dd58acdd849f0f6d3a06317fc9cd21e0b0e136c Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 17 Aug 2023 09:42:03 +0200 Subject: [PATCH 0033/1239] Set EVE_NTH_EVENT also for COSMICS (#1177) --- DATA/production/workflow-multiplicities.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index d370bf144..7420cf69c 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -201,6 +201,8 @@ if [[ -z ${EVE_NTH_EVENT:-} ]]; then EVE_NTH_EVENT=10 elif [[ $BEAMTYPE == "pp" && "${ED_VERTEX_MODE:-}" == "1" ]]; then EVE_NTH_EVENT=$((4 * 250 / $RECO_NUM_NODES_WORKFLOW_CMP)) + else # COSMICS + EVE_NTH_EVENT=10 fi [[ ! -z ${EPN_GLOBAL_SCALING:-} ]] && EVE_NTH_EVENT=$(($EVE_NTH_EVENT * $EPN_GLOBAL_SCALING)) fi From 35762fa525dea2b26119016e6e1636ff8a049d70 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 17 Aug 2023 09:46:26 +0200 Subject: [PATCH 0034/1239] Default should actually be one, though it deosn't really matter --- DATA/production/workflow-multiplicities.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 7420cf69c..56b1d66fc 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -201,8 +201,8 @@ if [[ -z ${EVE_NTH_EVENT:-} ]]; then EVE_NTH_EVENT=10 elif [[ $BEAMTYPE == "pp" && "${ED_VERTEX_MODE:-}" == "1" ]]; then EVE_NTH_EVENT=$((4 * 250 / $RECO_NUM_NODES_WORKFLOW_CMP)) - else # COSMICS - EVE_NTH_EVENT=10 + else # COSMICS / TECHNICALS / ... + EVE_NTH_EVENT=1 fi [[ ! -z ${EPN_GLOBAL_SCALING:-} ]] && EVE_NTH_EVENT=$(($EVE_NTH_EVENT * $EPN_GLOBAL_SCALING)) fi From 4c565b4754e6f8d03267f5f7b2def5025c1443a8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 21 Aug 2023 18:04:51 +0200 Subject: [PATCH 0035/1239] State which cached topology was used --- DATA/tools/epn/gen_topo_o2dpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 60f111aa5..881c4cf59 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -66,7 +66,7 @@ while true; do if [[ "0$GEN_TOPO_WIPE_CACHE" == "01" ]]; then rm -f cache/$GEN_TOPO_CACHE_HASH fi - echo Reusing cached XML topology 1>&2 + echo Reusing cached XML topology $GEN_TOPO_CACHE_HASH 1>&2 touch cache/$GEN_TOPO_CACHE_HASH cp cache/$GEN_TOPO_CACHE_HASH $GEN_TOPO_WORKDIR/output.xml break From 77aec2717ebd9c571a3ff2d5b1bc46508e2f7d87 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 21 Aug 2023 18:05:54 +0200 Subject: [PATCH 0036/1239] Use new topology merger by default --- DATA/tools/epn/gen_topo.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 564753259..516e7a87e 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -27,13 +27,16 @@ if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" else - export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone online --calibzone calib" + export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone online-mi50 --reco100zone online-mi100 --calibzone calib" fi fi [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://127.0.0.1:8084" # CCDB server to use if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi -#Temporary hacks +# Hack new topo-merger in +if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then + export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /scratch/services/topo_merger/topo-merger.py' +fi # GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running. if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then From 63ef070c45d09c820afb242266b7d9e9f19e2f61 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Mon, 21 Aug 2023 17:53:41 +0200 Subject: [PATCH 0037/1239] Exclude LHC23zt from 3 BC shift --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index fa099c8bc..96df4752c 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -82,8 +82,8 @@ CTP_BC_SHIFT=0 if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then CTP_BC_SHIFT=-294 fi -if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539908 ]]; then - # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zt +if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539700 ]]; then + # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zs CTP_BC_SHIFT=-3 fi if [[ $PERIOD == "LHC22s" ]]; then From 43a104ae01e518c9a7cd3a1e0611cb1e91b944d3 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Tue, 22 Aug 2023 10:41:13 +0200 Subject: [PATCH 0038/1239] Move export of CTP offset outside of dataset block Make sure it is applied for any dataset where it is requested. --- .../2022/LHC22f/apass1/setenv_extra.sh | 13 ++++++++++--- .../configurations/asyncReco/setenv_extra.sh | 9 ++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh index 55f5561c1..648e9a61b 100644 --- a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh @@ -72,8 +72,8 @@ fi CTP_BC_SHIFT=0 if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then CTP_BC_SHIFT=-294 -if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539908 ]]; then - # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zt +if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539700 ]]; then + # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zs CTP_BC_SHIFT=-3 fi if [[ $PERIOD == "LHC22s" ]]; then @@ -102,7 +102,7 @@ if [[ $PERIOD == "LHC22s" ]]; then fi CTP_BC_SHIFT=-293 if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE != "MC" ]]; then - export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[2]=1;TriggerOffsetsParam.customOffset[4]=1;TriggerOffsetsParam.customOffset[5]=1;TriggerOffsetsParam.customOffset[6]=1;TriggerOffsetsParam.customOffset[7]=1;TriggerOffsetsParam.customOffset[11]=$ZDC_BC_SHIFT;TriggerOffsetsParam.customOffset[16]=$CTP_BC_SHIFT" + export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[2]=1;TriggerOffsetsParam.customOffset[4]=1;TriggerOffsetsParam.customOffset[5]=1;TriggerOffsetsParam.customOffset[6]=1;TriggerOffsetsParam.customOffset[7]=1;TriggerOffsetsParam.customOffset[11]=$ZDC_BC_SHIFT" fi export PVERTEXER+=";pvertexer.dbscanDeltaT=1;pvertexer.maxMultRatDebris=1.;" fi @@ -135,6 +135,13 @@ if [[ $PERIOD == "LHC22q" ]]; then fi fi +# Apply BC shift of CTP IRs (whenever it is defined) +if [[ $CTP_BC_SHIFT -ne 0 ]]; then + if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE != "MC" ]]; then + export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[16]=$CTP_BC_SHIFT" + fi +fi + # ITSTPC vs FT0 time shift if [[ -z $TPCCLUSTERTIMESHIFT ]]; then SHIFTSCRIPT="$O2DPG_ROOT/DATA/production/configurations/$ALIEN_JDL_LPMANCHORYEAR/$O2DPGPATH/$PASS/ShiftMap.sh" diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 96df4752c..f1899812b 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -112,7 +112,7 @@ if [[ $PERIOD == "LHC22s" ]]; then fi CTP_BC_SHIFT=-293 if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE != "MC" ]]; then - export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[2]=1;TriggerOffsetsParam.customOffset[4]=1;TriggerOffsetsParam.customOffset[5]=1;TriggerOffsetsParam.customOffset[6]=1;TriggerOffsetsParam.customOffset[7]=1;TriggerOffsetsParam.customOffset[11]=$ZDC_BC_SHIFT;TriggerOffsetsParam.customOffset[16]=$CTP_BC_SHIFT" + export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[2]=1;TriggerOffsetsParam.customOffset[4]=1;TriggerOffsetsParam.customOffset[5]=1;TriggerOffsetsParam.customOffset[6]=1;TriggerOffsetsParam.customOffset[7]=1;TriggerOffsetsParam.customOffset[11]=$ZDC_BC_SHIFT" fi export PVERTEXER+=";pvertexer.dbscanDeltaT=1;pvertexer.maxMultRatDebris=1.;" fi @@ -145,6 +145,13 @@ if [[ $PERIOD == "LHC22q" ]]; then fi fi +# Apply BC shift of CTP IRs (whenever it is defined) +if [[ $CTP_BC_SHIFT -ne 0 ]]; then + if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE != "MC" ]]; then + export CONFIG_EXTRA_PROCESS_o2_ctf_reader_workflow+=";TriggerOffsetsParam.customOffset[16]=$CTP_BC_SHIFT" + fi +fi + # ITSTPC vs FT0 time shift if [[ -z $TPCCLUSTERTIMESHIFT ]]; then SHIFTSCRIPT="$O2DPG_ROOT/DATA/production/configurations/asyncReco/ShiftMap.sh" From 1226bddaadd74364717ac05d90363b094518b6dd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 09:08:57 +0200 Subject: [PATCH 0039/1239] GenTopo: remove remainder from obsolete old odc-epn-topo command --- DATA/tools/epn/gen_topo.sh | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 516e7a87e..04608f9cc 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -33,7 +33,7 @@ fi [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://127.0.0.1:8084" # CCDB server to use if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi -# Hack new topo-merger in +# Command for topology generation if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /scratch/services/topo_merger/topo-merger.py' fi @@ -43,15 +43,7 @@ if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then [[ "0$GEN_TOPO_RUN_HOME_TEST" != "01" ]] && [[ $WORKFLOWMODE != "print" ]] && { echo "ERROR: GEN_TOPO_RUN_HOME is only supported with WORKFLOWMODE=print!" 1>&2; exit 1; } else if [ "0$GEN_TOPO_ONTHEFLY" == "01" ]; then - # In case we run the on the fly generation on the EPN, we define which odc-epn-topo binary to use. - # Then we purge the modules, since the topology generation will load the O2PDPSuite with the O2 version that shall run, and that includes ODC. - # If there is already something of ODC or O2PDPSuite in the environment, we should remove it to avoid collisions. - # We set the odc-epn-topo command to be used explicitly though. - # Note this happens only in case of on the fly generation when we run online, in case of tests this is not needed. - if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_CMD=`which odc-epn-topo` - [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]] && { echo "ERROR: no odc-epn-topo in the path" 1>&2; exit 1; } - fi + # We purge the modules, since the topology generation will load the O2PDPSuite with the O2 version that shall run, and that includes ODC. module purge &> /dev/null fi fi From 149d6bc728366fb6a72bce065583d3fef98bb05a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 09:09:23 +0200 Subject: [PATCH 0040/1239] GenTopo: fix wiping of topology cache --- DATA/tools/epn/gen_topo_o2dpg.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 881c4cf59..785073e46 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -58,19 +58,20 @@ mkdir -p $GEN_TOPO_WORKDIR/cache || { echo Error creating directory 1>&2; exit 1 while true; do if [[ $GEN_TOPO_HASH == 1 ]]; then cd $GEN_TOPO_WORKDIR || { echo Cannot enter work dir 1>&2; exit 1; } - if [[ ! -d O2DPG ]]; then git clone https://github.com/AliceO2Group/O2DPG.git 1>&2 || { echo O2DPG checkout failed 1>&2; exit 1; }; fi if [[ "0$GEN_TOPO_ONTHEFLY" == "01" && ! -z "$GEN_TOPO_CACHE_HASH" ]]; then export GEN_TOPO_CACHEABLE=1 fi if [[ "0$GEN_TOPO_CACHEABLE" == "01" && -f cache/$GEN_TOPO_CACHE_HASH ]]; then if [[ "0$GEN_TOPO_WIPE_CACHE" == "01" ]]; then rm -f cache/$GEN_TOPO_CACHE_HASH + else + echo Reusing cached XML topology $GEN_TOPO_CACHE_HASH 1>&2 + touch cache/$GEN_TOPO_CACHE_HASH + cp cache/$GEN_TOPO_CACHE_HASH $GEN_TOPO_WORKDIR/output.xml + break fi - echo Reusing cached XML topology $GEN_TOPO_CACHE_HASH 1>&2 - touch cache/$GEN_TOPO_CACHE_HASH - cp cache/$GEN_TOPO_CACHE_HASH $GEN_TOPO_WORKDIR/output.xml - break fi + if [[ ! -d O2DPG ]]; then git clone https://github.com/AliceO2Group/O2DPG.git 1>&2 || { echo O2DPG checkout failed 1>&2; exit 1; }; fi cd O2DPG git checkout $GEN_TOPO_SOURCE &> /dev/null if [[ $? != 0 ]]; then From af499c6ee9aeaa27441ae266958b30382a84b1e1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 09:09:46 +0200 Subject: [PATCH 0041/1239] GenTopo: add option to run a postprocessing command after topology caching --- DATA/tools/epn/gen_topo.sh | 2 ++ DATA/tools/epn/gen_topo_o2dpg.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 04608f9cc..b7692646d 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -37,6 +37,8 @@ if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /scratch/services/topo_merger/topo-merger.py' fi +# Command for postprocessing of topology generation after topology caching +# GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='...' # GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running. if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 785073e46..781ee1572 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -107,6 +107,10 @@ while true; do break done +if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then + $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_WORKDIR/output.xml +fi + cat $GEN_TOPO_WORKDIR/output.xml echo Removing temporary output file $GEN_TOPO_WORKDIR/output.xml 1>&2 rm $GEN_TOPO_WORKDIR/output.xml From be34ad174d2e12a25e8a6067c4ca8ffbcd8a0630 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 09:16:09 +0200 Subject: [PATCH 0042/1239] GenTopo: store zipped topology file in log folder also for cached topologies. --- DATA/tools/epn/gen_topo_o2dpg.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 781ee1572..7bc3956ee 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -99,11 +99,6 @@ while true; do cp $GEN_TOPO_WORKDIR/output.xml cache/$GEN_TOPO_CACHE_HASH fi - if [[ ! -z "$ECS_ENVIRONMENT_ID" && -d "/var/log/topology/" && $USER == "epn" ]]; then - GEN_TOPO_LOG_FILE=/var/log/topology/topology-$(date -u +%Y%m%d-%H%M%S)-$ECS_ENVIRONMENT_ID.xml - cp $GEN_TOPO_WORKDIR/output.xml $GEN_TOPO_LOG_FILE - nohup gzip $GEN_TOPO_LOG_FILE &> /dev/null & - fi break done @@ -111,6 +106,12 @@ if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_WORKDIR/output.xml fi +if [[ ! -z "$ECS_ENVIRONMENT_ID" && -d "/var/log/topology/" && $USER == "epn" ]]; then + GEN_TOPO_LOG_FILE=/var/log/topology/topology-$(date -u +%Y%m%d-%H%M%S)-$ECS_ENVIRONMENT_ID.xml + cp $GEN_TOPO_WORKDIR/output.xml $GEN_TOPO_LOG_FILE + nohup gzip $GEN_TOPO_LOG_FILE &> /dev/null & +fi + cat $GEN_TOPO_WORKDIR/output.xml echo Removing temporary output file $GEN_TOPO_WORKDIR/output.xml 1>&2 rm $GEN_TOPO_WORKDIR/output.xml From 0453d57e4d10773f0801c33dc3c8db9684faae19 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Tue, 22 Aug 2023 11:42:01 +0200 Subject: [PATCH 0043/1239] [EMCAL-718, EMCAL-1037] Load dedicated configuration for different conditions - [EMCAL-718] Load dedicated configuration for beamtype PbPb - [EMCAL-1037] Add dedicated configuration for subsciption to CTP digits if CTP is in readout --- DATA/production/qc-workflow.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 703f6312f..dc557861f 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -63,7 +63,21 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_FDD:-}" ]] && QC_JSON_FDD=consul://o2/components/qc/ANY/any/fdd-digits-qc-epn [[ -z "${QC_JSON_FT0:-}" ]] && QC_JSON_FT0=consul://o2/components/qc/ANY/any/ft0-digits-qc-epn [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=consul://o2/components/qc/ANY/any/fv0-digits-qc-epn - [[ -z "${QC_JSON_EMC:-}" ]] && QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall + if [[ -z "${QC_JSON_EMC:-}" ]]; then + if [ "$BEAMTYPE" == "PbPb"]; then + if has_detector CTP; then + QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall-withCTP-PbPb + else + QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall-PbPb + fi + else + if has_detector CTP; then + QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall-withCTP + else + QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall + fi + fi + fi [[ -z "${QC_JSON_ZDC:-}" ]] && has_processing_step ZDC_RECO && QC_JSON_ZDC=consul://o2/components/qc/ANY/any/zdc-rec-epn if [[ -z "${QC_JSON_MCH:-}" ]]; then if has_detector MCH && has_processing_step MCH_RECO; then From 871280387266b1a08b0bbcbfb388249ad36b97f3 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 17:09:16 +0200 Subject: [PATCH 0044/1239] Must not use SHM for this transport for local tests, since inpuit proxies of different calib workflows will alter the message so they must not share it. --- DATA/common/setenv_calib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 95cb13a9d..1267e8b70 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -369,7 +369,7 @@ get_proxy_connection() fi if workflow_has_parameter CALIB_LOCAL_AGGREGATOR; then - CONNECTION+=",transport=shmem,address=ipc://${UDS_PREFIX}aggregator-shm-$1" + CONNECTION+=",transport=zeromq,address=ipc://${UDS_PREFIX}aggregator-shm-$1" else CONNECTION+=",transport=zeromq" fi From ca8814ee898cd09a034fbf5fe41d66bc412a9105 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 23 Aug 2023 23:52:54 +0200 Subject: [PATCH 0045/1239] Revert "calib-workflow: WORKAROUND: Currently cannot send CTP data to 2 output proxies" This reverts commit 3b1f236981f6edcc914bff424e897161402644ce. --- DATA/common/setenv_calib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 1267e8b70..edcd4f28f 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -271,7 +271,7 @@ if [[ -z ${CALIBDATASPEC_CALO_TF:-} ]]; then if [[ $CALIB_EMC_BADCHANNELCALIB == 1 ]] || [[ $CALIB_EMC_TIMECALIB == 1 ]]; then add_semicolon_separated CALIBDATASPEC_CALO_TF "cellsEMC:EMC/CELLS/0" add_semicolon_separated CALIBDATASPEC_CALO_TF "cellsTrgREMC:EMC/CELLSTRGR/0" - if false && has_detector CTP; then # FIXME: Currently we cannot send CTP/DIGITS to both CALO and BARREL workflow. + if has_detector CTP; then add_semicolon_separated CALIBDATASPEC_CALO_TF "ctpdigi:CTP/DIGITS/0" fi fi From 597ac26c10b604710a3db145a2aefd3e3e758216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubo=C5=A1=20Kr=C4=8D=C3=A1l?= Date: Wed, 23 Aug 2023 10:03:16 +0200 Subject: [PATCH 0046/1239] Use --assets in EPN topology merger instead of sed --- DATA/tools/parse | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/DATA/tools/parse b/DATA/tools/parse index 101b70c28..1ddc7b8af 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -258,11 +258,18 @@ for line in f: for i in env_at_runtime: odccommand += 'export ' + i + '=' + os.environ[i] + ' ; ' odccommand += '"' + if os.path.exists(os.environ['GEN_TOPO_QC_JSON_FILE']): + odccommand += f" --assets {os.environ['GEN_TOPO_QC_JSON_FILE']}" odccommand += ' -o ' + sys.argv[3] - print('Running ODC command', odccommand) + print('Running topology merger command', odccommand) starttime = time.time() - if os.system(odccommand) != 0: - print('\nError running odc: ', odccommand) + try: + out = subprocess.check_output(odccommand, stderr=subprocess.STDOUT, shell=True) + print(out.decode('utf-8')) + print('Topology merger finished successfully') + except subprocess.CalledProcessError as e: + print(e.output.decode('utf-8')) + print('Error running topology merger!') raise print('Execution time: ', time.time() - starttime) if os.path.exists(os.environ['GEN_TOPO_QC_JSON_FILE']): @@ -270,16 +277,6 @@ for line in f: print('Running SED command for DPL JSON XML Escaping', command) if os.system(command) != 0: print('Error running sed on JSON file') - command = 'sed -i ' + \ - '-e \'s,' + os.environ['GEN_TOPO_QC_JSON_FILE'] + ',${DDS_LOCATION}/qc.json.asset,\' ' + \ - '-e \'/^ *$/a qc.json\' ' + \ - '-e \'/^ *$/a \' ' + \ - sys.argv[3] - print('Running final SED command for QC JSONs', command) - if os.system(command) != 0: - print('Error running sed') else: outf = open(sys.argv[3], 'w+') for i in recoworkflows: From 17811ca2e45d22572c035a5ceeaf988d8c5a4bff Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Wed, 23 Aug 2023 18:19:53 +0200 Subject: [PATCH 0047/1239] [EMCAL-1037] Include EMCAL BC moniotoring in async QC --- DATA/production/qc-async/emc.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/DATA/production/qc-async/emc.json b/DATA/production/qc-async/emc.json index 4b6dd8d1e..dfdfbb4c5 100644 --- a/DATA/production/qc-async/emc.json +++ b/DATA/production/qc-async/emc.json @@ -62,6 +62,21 @@ "hasInvMassMesons": "true", "mesonClustersRejectExotics": 1 } + }, + "BCTaskEMCAL": { + "active": "true", + "taskName": "BCs", + "className": "o2::quality_control_modules::emcal::BCTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-triggers:EMC/CELLSTRGR;ctp-digits:CTP/DIGITS" + }, + "taskParameters": { + } } } }, From 7c5e154f3bb1b62d5ade755ac13aab5ffe8c88d4 Mon Sep 17 00:00:00 2001 From: iravasen Date: Mon, 21 Aug 2023 10:18:42 +0200 Subject: [PATCH 0048/1239] push to ccdb test for synthetic/staging calibration scans --- .../calib/its-threshold-aggregator.sh | 9 +++++++-- .../calib/its-threshold-processing.sh | 10 +++++----- DATA/production/standalone-calibration.desc | 18 +++++++++--------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/DATA/production/calib/its-threshold-aggregator.sh b/DATA/production/calib/its-threshold-aggregator.sh index 5fdebfc55..572a9fb79 100755 --- a/DATA/production/calib/its-threshold-aggregator.sh +++ b/DATA/production/calib/its-threshold-aggregator.sh @@ -10,17 +10,22 @@ PROXY_INSPEC="tunestring:ITS/TSTR;runtype:ITS/RUNT;fittype:ITS/FITT;scantype:ITS CCDBPATH1="" CCDBPATH2="" -if [ $RUNTYPE == "tuning" ] || [ $RUNTYPE == "digital" ] || [ $RUNTYPE == "tuningbb" ]; then +if [ $RUNTYPE_ITS == "tuning" ] || [ $RUNTYPE_ITS == "digital" ] || [ $RUNTYPE_ITS == "tuningbb" ]; then CCDBPATH1="http://alio2-cr1-flp199.cern.ch:8083" CCDBPATH2="http://o2-ccdb.internal" else CCDBPATH1="http://o2-ccdb.internal" fi +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + CCDBPATH1="http://ccdb-test.cern.ch:8080" + CCDBPATH2="http://ccdb-test.cern.ch:8080" +fi + WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --exit-transition-timeout 20 --proxy-name its-thr-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-thr-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " WORKFLOW+="o2-its-threshold-aggregator-workflow -b $ARGS_ALL | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$CCDBPATH1\" --sspec-min 0 --sspec-max 0 --name-extention dcs | " -if [ $RUNTYPE == "digital" ]; then +if [ $RUNTYPE_ITS == "digital" ]; then WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$CCDBPATH2\" --sspec-min 1 --sspec-max 1 | " fi diff --git a/DATA/production/calib/its-threshold-processing.sh b/DATA/production/calib/its-threshold-processing.sh index 2214e2479..ce2a04844 100755 --- a/DATA/production/calib/its-threshold-processing.sh +++ b/DATA/production/calib/its-threshold-processing.sh @@ -14,23 +14,23 @@ PROXY_OUTSPEC="tunestring:ITS/TSTR;runtype:ITS/RUNT;fittype:ITS/FITT;scantype:IT CHIPMODBASE=5 NDECODERS=6 -if [ $RUNTYPE == "digital" ]; then +if [ $RUNTYPE_ITS == "digital" ]; then CHIPMODBASE=10 fi -if [ $RUNTYPE == "thrfull" ]; then +if [ $RUNTYPE_ITS == "thrfull" ]; then CHIPMODBASE=20 NDECODERS=10 fi ADDITIONAL_OPTIONS_DEC="" ADDITIONAL_OPTIONS_CAL="" -if [ $RUNTYPE == "tuningbb" ]; then +if [ $RUNTYPE_ITS == "tuningbb" ]; then ADDITIONAL_OPTIONS_CAL="--min-vcasn 30 --max-vcasn 130" fi -if [[ $RUNTYPE == "tot1row" || $RUNTYPE == "vresetd" ]]; then +if [[ $RUNTYPE_ITS == "tot1row" || $RUNTYPE_ITS == "vresetd" ]]; then ADDITIONAL_OPTIONS_DEC="--allow-empty-rofs" ADDITIONAL_OPTIONS_CAL="--ninj 10" fi -if [ $RUNTYPE == "totfullfast" ]; then +if [ $RUNTYPE_ITS == "totfullfast" ]; then ADDITIONAL_OPTIONS_DEC="--allow-empty-rofs" ADDITIONAL_OPTIONS_CAL="--calculate-slope --charge-a 30 --charge-b 60 --ninj 10" fi diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index 885bad4e8..03b70b53d 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -4,23 +4,23 @@ ITS-noise-calibration: "O2PDPSuite" reco,20,20,"NITSDECTHREADS=4 NITSDECTPIPELIN ITS-noise-calibration-clusters: "O2PDPSuite" reco,20,20,"NITSDECTHREADS=4 NITSDECTPIPELINES=6 USECLUSTERS=1 production/calib/its-noise-processing.sh" calib,20,"USECLUSTERS=1 NTHREADS=32 production/calib/its-noise-aggregator.sh" -ITS-thr-tuning: "O2PDPSuite" reco,40,40,"RUNTYPE=tuning production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=tuning production/calib/its-threshold-aggregator.sh" +ITS-thr-tuning: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=tuning production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=tuning production/calib/its-threshold-aggregator.sh" -ITS-thr-tuning-bb: "O2PDPSuite" reco,40,40,"RUNTYPE=tuningbb production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=tuningbb production/calib/its-threshold-aggregator.sh" +ITS-thr-tuning-bb: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=tuningbb production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=tuningbb production/calib/its-threshold-aggregator.sh" -ITS-thr-short: "O2PDPSuite" reco,40,40,"RUNTYPE=thrshort production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=thrshort production/calib/its-threshold-aggregator.sh" +ITS-thr-short: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=thrshort production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=thrshort production/calib/its-threshold-aggregator.sh" -ITS-thr-full: "O2PDPSuite" reco,80,80,"RUNTYPE=thrfull production/calib/its-threshold-processing.sh" calib,80,"RUNTYPE=thrfull production/calib/its-threshold-aggregator.sh" +ITS-thr-full: "O2PDPSuite" reco,80,80,"RUNTYPE_ITS=thrfull production/calib/its-threshold-processing.sh" calib,80,"RUNTYPE_ITS=thrfull production/calib/its-threshold-aggregator.sh" -ITS-digital: "O2PDPSuite" reco,40,40,"RUNTYPE=digital production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=digital production/calib/its-threshold-aggregator.sh" +ITS-digital: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=digital production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=digital production/calib/its-threshold-aggregator.sh" -ITS-pulselength: "O2PDPSuite" reco,40,40,"RUNTYPE=pulselength production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=pulselength production/calib/its-threshold-aggregator.sh" +ITS-pulselength: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=pulselength production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=pulselength production/calib/its-threshold-aggregator.sh" -ITS-tot-1row: "O2PDPSuite" reco,40,40,"RUNTYPE=tot1row production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=tot1row production/calib/its-threshold-aggregator.sh" +ITS-tot-1row: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=tot1row production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=tot1row production/calib/its-threshold-aggregator.sh" -ITS-tot-fullfast: "O2PDPSuite" reco,40,40,"RUNTYPE=totfullfast production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE=totfullfast production/calib/its-threshold-aggregator.sh" +ITS-tot-fullfast: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=totfullfast production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=totfullfast production/calib/its-threshold-aggregator.sh" -ITS-vresetd: "O2PDPSuite" reco,5,5,"RUNTYPE=vresetd production/calib/its-threshold-processing.sh" calib,5,"RUNTYPE=vresetd production/calib/its-threshold-aggregator.sh" +ITS-vresetd: "O2PDPSuite" reco,5,5,"RUNTYPE_ITS=vresetd production/calib/its-threshold-processing.sh" calib,5,"RUNTYPE_ITS=vresetd production/calib/its-threshold-aggregator.sh" TOF-diagnostic-calibration: "O2PDPSuite" reco,10,10,"SHMSIZE=120376524800 production/calib/tof-standalone-reco.sh" calib,4,"production/calib/tof-diagn-aggregator.sh" From d9c014576e3d57f8efdf0d0f967246ce9731a7d5 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:06:20 +0200 Subject: [PATCH 0049/1239] TPC: Add laser raw filter workflow and pattern generator workflow to testing folder (#1179) * Add laser raw-filter workflow and move pattern_generator workflow to TPC/test * laser raw-filter cleanup --- DATA/production/standalone-calibration.desc | 2 - .../detectors/TPC/tpc-laser-raw-filter.sh | 45 ++++++++++ DATA/testing/detectors/TPC/tpc-laser.sh | 86 ------------------- .../detectors/TPC}/tpc-pattern-generator.sh | 0 DATA/testing/detectors/TPC/tpc-pedestal.sh | 45 ---------- DATA/testing/detectors/TPC/tpc-pulser.sh | 31 ------- DATA/testing/detectors/TPC/tpc-standalone.sh | 71 --------------- DATA/testing/detectors/TPC/tpc-workflow.sh | 60 ------------- DATA/testing/detectors/TPC/workflows.desc | 9 +- 9 files changed, 47 insertions(+), 302 deletions(-) create mode 100755 DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh delete mode 100755 DATA/testing/detectors/TPC/tpc-laser.sh rename DATA/{production/calib => testing/detectors/TPC}/tpc-pattern-generator.sh (100%) delete mode 100755 DATA/testing/detectors/TPC/tpc-pedestal.sh delete mode 100755 DATA/testing/detectors/TPC/tpc-pulser.sh delete mode 100755 DATA/testing/detectors/TPC/tpc-standalone.sh delete mode 100755 DATA/testing/detectors/TPC/tpc-workflow.sh diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index 03b70b53d..6cb943d4d 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -36,8 +36,6 @@ TPC-laser: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE= TPC-laser-multi: "O2PDPSuite" reco,10,10,"SHMSIZE=$((64 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser-filter.sh" calib,1,"production/calib/tpc-laser-aggregator.sh" -TPC-pattern-generator: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-pattern-generator.sh" - MFT-noise-calibration: "O2PDPSuite" reco,20,20,"production/calib/mft-noise-processing.sh" calib,20,"production/calib/mft-noise-aggregator.sh" MCH-badchannel-calibration: "O2PDPSuite" reco,5,5,"production/calib/mch-badchannel-processing.sh" calib,20,"production/calib/mch-badchannel-aggregator.sh" diff --git a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh new file mode 100755 index 000000000..39a4a3deb --- /dev/null +++ b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +source common/setenv.sh + +source common/getCommonArgs.sh + +export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder + +if [ $NUMAGPUIDS != 0 ]; then + ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" +fi + +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" + +NLANES=36 +SESSION="default" +PIPEADD="0" +ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" + +HOST=localhost + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + + +o2-dpl-raw-proxy $ARGS_ALL \ + --dataspec "$PROXY_INSPEC" \ + --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ + | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ + --input-spec "$CALIB_INSPEC" \ + --configKeyValues "$ARGS_FILES" \ + --remove-duplicates \ + --pipeline tpc-raw-to-digits-0:24 \ + | o2-tpc-krypton-raw-filter $ARGS_ALL \ + --configKeyValues "$ARGS_FILES" \ + --lanes $NLANES \ + --writer-type EPN \ + --meta-output-dir $EPN2EOS_METAFILES_DIR \ + --output-dir $CALIB_DIR \ + --threshold-max 20 \ + --max-tf-per-file 8000 \ + --time-bins-before 20 \ + --max-time-bins 650 \ + | o2-qc $ARGS_ALL --config ${QC_CONFIG} --local --host $HOST \ + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/testing/detectors/TPC/tpc-laser.sh b/DATA/testing/detectors/TPC/tpc-laser.sh deleted file mode 100755 index 67d8bd65d..000000000 --- a/DATA/testing/detectors/TPC/tpc-laser.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -source common/setenv.sh - -export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM -export GPUMEMSIZE=$(( 24 << 30 )) -export HOSTMEMSIZE=$(( 5 << 30 )) -export GPUTYPE="HIP" - -FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" - -FILEWORKDIR2="/home/epn/odc/files/" - -source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi - -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;" - GPU_CONFIG+=" --environment ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi - -if [ $GPUTYPE != "CPU" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then - HOSTMEMSIZE=$(( 1 << 30 )) - fi -fi -if [ $HOSTMEMSIZE != "0" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" -fi - -#source /home/epn/runcontrol/tpc/qc_test_env.sh > /dev/null -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" - -### Comment: MAKE SURE the channels match address=ipc://@tf-builder-pipe-0 - -#VERBOSE="" - -#echo GPU_CONFIG $GPU_CONFIG_KEYS; - - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \ - --pipeline tpc-raw-to-digits-0:32 \ - --remove-duplicates \ - --send-ce-digits \ - | o2-tpc-reco-workflow $ARGS_ALL \ - --input-type digitizer \ - --output-type "tracks,disable-writer" \ - --disable-mc \ - --pipeline tpc-tracker:4 \ - $GPU_CONFIG \ - --configKeyValues "$ARGS_ALL_CONFIG;align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY" \ - | o2-tpc-laser-track-filter $ARGS_ALL \ - | o2-tpc-calib-laser-tracks $ARGS_ALL --use-filtered-tracks --min-tfs 50 \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --configKeyValues "TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=150;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=302;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" \ - --lanes 36 \ - --calib-type ce \ - --publish-after-tfs 50 \ - --max-events 90 \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - --ccdb-path http://ccdb-test.cern.ch:8080 \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} - - -# --configKeyValues "align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;GPU_proc.deviceNum=0;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$ARGS_FILES;keyval.output_dir=/dev/null" \ diff --git a/DATA/production/calib/tpc-pattern-generator.sh b/DATA/testing/detectors/TPC/tpc-pattern-generator.sh similarity index 100% rename from DATA/production/calib/tpc-pattern-generator.sh rename to DATA/testing/detectors/TPC/tpc-pattern-generator.sh diff --git a/DATA/testing/detectors/TPC/tpc-pedestal.sh b/DATA/testing/detectors/TPC/tpc-pedestal.sh deleted file mode 100755 index 61b38ec95..000000000 --- a/DATA/testing/detectors/TPC/tpc-pedestal.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -source common/setenv.sh - -source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi - -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi - -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;GPU_global.mutexMemReg=true;" - GPU_CONFIG+=" --environment \"ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}\"" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" - - -CALIB_CONFIG="TPCCalibPedestal.LastTimeBin=12000" -EXTRA_CONFIG=" " -EXTRA_CONFIG=" --publish-after-tfs 100 --max-events 120 --lanes 36" -CCDB_PATH="--ccdb-path http://ccdb-test.cern.ch:8080" - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$CALIB_CONFIG;keyval.output_dir=/dev/null" \ - $EXTRA_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - $CCDB_PATH \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/testing/detectors/TPC/tpc-pulser.sh b/DATA/testing/detectors/TPC/tpc-pulser.sh deleted file mode 100755 index 6f98ee9e4..000000000 --- a/DATA/testing/detectors/TPC/tpc-pulser.sh +++ /dev/null @@ -1,31 +0,0 @@ - -#!/usr/bin/env bash - -source common/setenv.sh - -source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi - -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" - -CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=160;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=115;TPCCalibPulser.XmaxT0=130;TPCCalibPulser.NbinsT0=600" - -EXTRA_CONFIG=" " -EXTRA_CONFIG="--calib-type pulser --publish-after-tfs 1000 --max-events 1200 --lanes 36" - -CCDB_PATH="--ccdb-path http://ccdb-test.cern.ch:8080" - - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$CALIB_CONFIG;keyval.output_dir=/dev/null" \ - $EXTRA_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - $CCDB_PATH \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/testing/detectors/TPC/tpc-standalone.sh b/DATA/testing/detectors/TPC/tpc-standalone.sh deleted file mode 100755 index 30e83354b..000000000 --- a/DATA/testing/detectors/TPC/tpc-standalone.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -source common/setenv.sh - -export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM -export GPUMEMSIZE=$(( 24 << 30 )) -export HOSTMEMSIZE=$(( 5 << 30 )) - - -FILEWORKDIR="/home/epn/odc/files/" - -source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi - -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;" - GPU_CONFIG+=" --environment ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi - -if [ $GPUTYPE != "CPU" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then - HOSTMEMSIZE=$(( 1 << 30 )) - fi -fi -if [ $HOSTMEMSIZE != "0" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" -fi - -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" - -### Comment: MAKE SURE the channels match address=ipc://@tf-builder-pipe-0 - -NCPU=12 #$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}') -ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" -#HOST='$(hostname -s)-ib' - -HOST=localhost - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "TPCDigitDump.LastTimeBin=1000;$ARGS_ALL_CONFIG" \ - --remove-duplicates \ - --pipeline tpc-raw-to-digits-0:32 \ - | o2-tpc-reco-workflow $ARGS_ALL \ - --input-type digitizer \ - --output-type clusters,tracks,disable-writer \ - --disable-mc \ - --pipeline tpc-tracker:8 \ - $GPU_CONFIG \ - --configKeyValues "$ARGS_ALL_CONFIG;align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY" \ - | o2-qc $ARGS_ALL --config consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-full-qcmn --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} | grep -v ERROR diff --git a/DATA/testing/detectors/TPC/tpc-workflow.sh b/DATA/testing/detectors/TPC/tpc-workflow.sh deleted file mode 100755 index 27004076a..000000000 --- a/DATA/testing/detectors/TPC/tpc-workflow.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -source common/setenv.sh - -source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi - -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;GPU_global.mutexMemReg=true;" - GPU_CONFIG+=" --environment \"ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}\"" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi - -if [ $GPUTYPE != "CPU" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then - HOSTMEMSIZE=$(( 1 << 30 )) - fi -fi -if [ $HOSTMEMSIZE != "0" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" -fi - -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --remove-duplicates \ - --pipeline tpc-raw-to-digits-0:6 \ - --configKeyValues "$ARGS_ALL_CONFIG;TPCDigitDump.LastTimeBin=1000;" \ - | o2-tpc-reco-workflow $ARGS_ALL \ - --input-type digitizer \ - --output-type clusters,tracks,encoded-clusters disable-writer \ - --disable-mc \ - --pipeline tpc-tracker:4 \ - $GPU_CONFIG \ - --configKeyValues "$ARGS_ALL_CONFIG;$GPU_CONFIG_KEY;align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;" \ - | o2-eve-display $ARGS_ALL --display-tracks TPC --display-clusters TPC --disable-mc --jsons-folder /home/ed/jsons --eve-dds-collection-index 0 --configKeyValues "$ARGS_ALL_CONFIG" \ - | o2-ctf-writer-workflow $ARGS_ALL --configKeyValues "$ARGS_ALL_CONFIG" --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf --onlyDet TPC \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} - -#HOST=localhost -#| o2-qc $ARGS_ALL --config json:///home/epn/odc/files/tpcQCTasks_multinode_ALL.json --local --host $HOST \ diff --git a/DATA/testing/detectors/TPC/workflows.desc b/DATA/testing/detectors/TPC/workflows.desc index 647275bb9..1e763d763 100644 --- a/DATA/testing/detectors/TPC/workflows.desc +++ b/DATA/testing/detectors/TPC/workflows.desc @@ -1,8 +1,3 @@ -ctf-and-display: "DataDistribution QualityControl" reco,128,128,"SHMSIZE=128000000000 INFOLOGGER_SEVERITY=warning testing/detectors/TPC/tpc-workflow.sh" -ctf-and-display-gpu: "DataDistribution QualityControl" reco,128,128,"GPUTYPE=HIP SHMSIZE=128000000000 INFOLOGGER_SEVERITY=warning testing/detectors/TPC/tpc-workflow.sh" -TPC-pulser: "O2PDPSuite" reco,1,1," production/calib/tpc-pulser.sh" -TPC-pedestal: "O2PDPSuite" reco,1,1," production/calib/tpc-pedestal.sh" -TPC-laser: "O2PDPSuite" reco,1,1,"SHMSIZE=$((64 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser.sh" TPC-krypton: "O2PDPSuite" reco,60,60,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-krypton.sh" -TPC-krypton-raw: "O2PDPSuite" reco,80,80,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-krypton-raw.sh" -TPC-standalone: "O2PDPSuite" reco,60,60,"GPUTYPE=HIP SHMSIZE=128000000000 testing/detectors/TPC/tpc-standalone.sh" +TPC-laser-raw-filter: "O2PDPSuite" reco,80,80,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-laser-raw-filter.sh" +TPC-pattern-generator: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) testing/detectors/TPC/tpc-pattern-generator.sh" From 9c4e4bb3e84a5a82faf5b24e73b0733627e8ec16 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 26 Aug 2023 20:37:41 +0200 Subject: [PATCH 0050/1239] dpl-workflow: fix automatic process scaling --- DATA/common/gen_topo_helper_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index e8ae886ce..8f5afb177 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -177,7 +177,7 @@ get_N() # USAGE: get_N [processor-name] [DETECTOR_NAME] [RAW|CTF|REST] [threads, local NAME_DEFAULT="N_${5:-}" local MULT=${!NAME_PROC:-$((${!NAME_FACTOR} * ${!NAME_DET:-1} * ${!NAME_PROC_FACTOR:-1} * ${!NAME_DEFAULT:-1}))} [[ ! -z ${EPN_GLOBAL_SCALING:-} && $1 != "gpu-reconstruction" ]] && MULT=$(($MULT * $EPN_GLOBAL_SCALING)) - if [[ -z ${NAME_PROC} && "0$GEN_TOPO_AUTOSCALE_PROCESSES" == "01" && ($WORKFLOWMODE != "print" || $GEN_TOPO_RUN_HOME_TEST == 1) && $4 != 0 ]]; then + if [[ ${GEN_TOPO_AUTOSCALE_PROCESSES_GLOBAL_WORKFLOW:-} == 1 && -z ${!NAME_PROC:-} && "0$GEN_TOPO_AUTOSCALE_PROCESSES" == "01" && ($WORKFLOWMODE != "print" || $GEN_TOPO_RUN_HOME_TEST == 1) && $4 != 0 ]]; then echo $1:\$\(\(\($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) \< 16 ? \($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) : 16\)\) else echo $1:$MULT From 84d137c5215b2616694c9b105a90cedcc1cb67b2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 24 Aug 2023 10:39:35 +0200 Subject: [PATCH 0051/1239] Switch to using new 2-stage topology merger tool --- DATA/README.md | 3 ++- DATA/tools/epn/gen_topo.sh | 23 ++++++++++++++--------- DATA/tools/epn/gen_topo_o2dpg.sh | 17 +++++++++++++++-- DATA/tools/parse | 10 +--------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/DATA/README.md b/DATA/README.md index bc59da599..6b3bffbb7 100644 --- a/DATA/README.md +++ b/DATA/README.md @@ -52,7 +52,7 @@ There are 3 ways foreseen to configure the *full topology* in AliECS: (currently - The **workflow name** inside the *description library file*. - **detector list**: Multiple comma-separated lists of detectors participating in the run (global list, list for qc, list for calibration, list of detectors to run reconstruction for, list of detectors to include in the CTF, list of detectors that have processing on the FLP), defaulting to `ALL` for all detectors. - **workflow parameters**: text field passed to workflow as environment variable for additional options. - - **number of nodes override**: Overrides the setting for the number of nodes required in the workflow (meant to quickly increase / decrease the EPN partition size). + - **number of nodes override**: Overrides the setting for the number of nodes required in the workflow (meant to quickly increase / decrease the EPN partition size). **NOTE: This setting has become mandatory now, and is used exclusively to set the number of nodes. The number specified in the workflow description is ignored** - **process multiplicity overrides**: Scaling factors for the process multiplicities for raw decoders, ctf encoders, and other processes. - **extra environment options**: Free text field where the operator can put additional environment variables, that will be forwarded to the workflow. - **wipe workflow cache**: Normally the XMLs are cached, when they are created from the same repository version / same workflow / same O2 version. This option clears the cache for the current partition. @@ -67,6 +67,7 @@ A *topology description* consists of - Zone where to run the workflow (calib / reco) - For reco: - Number of nodes to run this workflow on + - **NOTE: This setting for the number of nodes is ignored now. Please use the number of nodes override in ECS!** - If a processor in the workflow needs to identify on which node it is running on, it can use the `$DDS_COLLECTION_INDEX` emvironment variable. - Minimum number of nodes required forthe workflow (in case of node failure) - In case the there are multiple workflows in the topology description, the largest number of nodes, and the largest minimum number of nodes are used. diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index b7692646d..35e3c571e 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -23,22 +23,27 @@ else [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=$HOME/gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. mkdir -p $HOME/gen_topo fi -if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then - if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" - else - export GEN_TOPO_ODC_EPN_TOPO_ARGS="--recozone online-mi50 --reco100zone online-mi100 --calibzone calib" - fi -fi [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://127.0.0.1:8084" # CCDB server to use if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi # Command for topology generation if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /scratch/services/topo_merger/topo-merger.py' + export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /home/lkrcal/epn/topogen/topo_merger.py' fi # Command for postprocessing of topology generation after topology caching -# GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='...' +export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /home/lkrcal/epn/topogen/topo_resource_alloc.py' + +# Extra arguments for topology generation +#if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then +# GEN_TOPO_ODC_EPN_TOPO_ARGS= +#fi +if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" ]]; then + if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" + else + export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS="--recozone online-mi50 --reco100zone online-mi100 --calibzone calib" + fi +fi # GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running. if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 7bc3956ee..e9aa5a03c 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -14,6 +14,7 @@ if [[ -z "${WORKFLOW_DETECTORS_QC+x}" && -z "${WORKFLOW_DETECTORS_EXCLUDE_QC+x}" if [[ -z "${WORKFLOW_DETECTORS_CALIB+x}" && -z "${WORKFLOW_DETECTORS_EXCLUDE_CALIB+x}" ]]; then echo \$WORKFLOW_DETECTORS_EXCLUDE_CALIB missing; exit 1; fi # Comma-separated list of detectors to run calibration for if [[ -z "${WORKFLOW_PARAMETERS+x}" ]]; then echo \$WORKFLOW_PARAMETERS missing; exit 1; fi # Additional parameters for workflow if [[ -z "${RECO_NUM_NODES_OVERRIDE+x}" ]]; then echo \$RECO_NUM_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes +if [[ -z "${RECO_MAX_FAIL_NODES_OVERRIDE+x}" ]]; then echo \$RECO_MAX_FAIL_NODES_OVERRIDE missing; exit 1; fi # Override number of nodes allowed to fail if [[ -z "$DDMODE" ]] && [[ -z "$DDWORKFLOW" ]]; then echo Either \$DDMODE or \$DDWORKFLOW must be set; exit 1; fi # Select data distribution workflow if [[ -z "$MULTIPLICITY_FACTOR_RAWDECODERS" ]]; then echo \$MULTIPLICITY_FACTOR_RAWDECODERS missing; exit 1; fi # Process multiplicity scaling parameter if [[ -z "$MULTIPLICITY_FACTOR_CTFENCODERS" ]]; then echo \$MULTIPLICITY_FACTOR_CTFENCODERS missing; exit 1; fi # Process multiplicity scaling parameter @@ -28,7 +29,7 @@ if [[ -z "$CTF_DIR" ]]; then echo \$CTF_DIR missing; exit 1; fi if [[ -z "$CALIB_DIR" ]]; then echo \$CALIB_DIR missing; exit 1; fi if [[ -z "$EPN2EOS_METAFILES_DIR" ]]; then echo \$EPN2EOS_METAFILES_DIR missing; exit 1; fi if [[ -z "$GEN_TOPO_WORKDIR" ]]; then echo \$GEN_TOPO_WORKDIR missing; exit 1; fi -if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then echo \$GEN_TOPO_ODC_EPN_TOPO_ARGS missing; exit 1; fi +if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then echo \$GEN_TOPO_ODC_EPN_TOPO_CMD missing; exit 1; fi if [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]]; then echo \$GEN_TOPO_EPN_CCDB_SERVER missing; exit 1; fi # Replace TRG by CTP @@ -103,7 +104,19 @@ while true; do done if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then - $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_WORKDIR/output.xml + TMP_POST_CACHING_CMD="$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" + TMP_POST_CACHING_NMIN=$(( $RECO_NUM_NODES_OVERRIDE > $RECO_MAX_FAIL_NODES_OVERRIDE ? $RECO_NUM_NODES_OVERRIDE - $RECO_MAX_FAIL_NODES_OVERRIDE : 0 )) + TMP_POST_CACHING_CMD+=" --nodes-mi50 $RECO_NUM_NODES_OVERRIDE --nmin-mi50 $TMP_POST_CACHING_NMIN" + if [[ -z $GEN_TOPO_MI100_NODES || $GEN_TOPO_MI100_NODES == "0" ]]; then + TMP_POST_CACHING_CMD+=" --force-exact-node-numbers --nodes-mi100 0 --nmin-mi100 0" + elif [[ ! -z $GEN_TOPO_MI100_NODES && $GEN_TOPO_MI100_NODES != "-1" ]]; then + TMP_POST_CACHING_NMIN=$(( $GEN_TOPO_MI100_NODES > $RECO_MAX_FAIL_NODES_OVERRIDE ? $GEN_TOPO_MI100_NODES - $RECO_MAX_FAIL_NODES_OVERRIDE : 0 )) + TMP_POST_CACHING_CMD+=" --force-exact-node-numbers --nodes-mi100 $GEN_TOPO_MI100_NODES --nmin-mi100 $TMP_POST_CACHING_NMIN" + fi + TMP_POST_CACHING_CMD+=" -o $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml" + echo "Running post-caching topo-merger command: $TMP_POST_CACHING_CMD" 1>&2 + eval $TMP_POST_CACHING_CMD 1>&2 || { echo Error during EPN topology-merger resource allocation 1>&2; exit 1; } + mv -f $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml 1>&2 fi if [[ ! -z "$ECS_ENVIRONMENT_ID" && -d "/var/log/topology/" && $USER == "epn" ]]; then diff --git a/DATA/tools/parse b/DATA/tools/parse index 1ddc7b8af..726991fe7 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -220,7 +220,7 @@ for line in f: odccommand = 'odc-epn-topo' if 'GEN_TOPO_ODC_EPN_TOPO_ARGS' in os.environ: odccommand += ' ' + os.environ['GEN_TOPO_ODC_EPN_TOPO_ARGS'] - if reconodes: + if True: replacestring = '' dd_env_variables = ['DD_DISK_FRACTION', 'EPN_DD_TEST', 'EPN_DD_TEST_2', 'EPN_DD_TEST_3', 'SHM_MANAGER_SHMID'] for i in dd_env_variables: @@ -238,14 +238,6 @@ for line in f: odccommand += ' --reco ' + ' '.join(recoworkflows) if len(recoworkflows_mi100): odccommand += ' --reco100 ' + ' '.join(recoworkflows_mi100) - odccommand += ' --n ' + str(reconodes) - odccommand += ' --nmin ' + str(reconodesmin) # Currently disabled, since odc-epn-topo does not accept --nmin - if 'GEN_TOPO_MI100_NODES' in os.environ and int(os.environ['GEN_TOPO_MI100_NODES']) > 0: - if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != '': - reconodesmin100 = max(1, int(os.environ['GEN_TOPO_MI100_NODES']) - int(os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'])) - else: - reconodesmin100 = min(int(os.environ['GEN_TOPO_MI100_NODES']), reconodesmin) - odccommand += ' --force-exact-node-numbers --n100 ' + os.environ['GEN_TOPO_MI100_NODES'] + ' --nmin100 ' + str(reconodesmin100) if len(calibworkflows): calibworkflowsdds.sort(key=lambda x:int(x.split(':')[1])*-1) odccommand += ' --calib ' + ' '.join(calibworkflowsdds) From 91bbc79a2800b24470f5f8672967cb03971dd9cf Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 27 Aug 2023 10:16:55 +0200 Subject: [PATCH 0052/1239] dpl-workflow: fix some undefined variables --- DATA/common/gen_topo_helper_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index 8f5afb177..e63313b6f 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -177,7 +177,7 @@ get_N() # USAGE: get_N [processor-name] [DETECTOR_NAME] [RAW|CTF|REST] [threads, local NAME_DEFAULT="N_${5:-}" local MULT=${!NAME_PROC:-$((${!NAME_FACTOR} * ${!NAME_DET:-1} * ${!NAME_PROC_FACTOR:-1} * ${!NAME_DEFAULT:-1}))} [[ ! -z ${EPN_GLOBAL_SCALING:-} && $1 != "gpu-reconstruction" ]] && MULT=$(($MULT * $EPN_GLOBAL_SCALING)) - if [[ ${GEN_TOPO_AUTOSCALE_PROCESSES_GLOBAL_WORKFLOW:-} == 1 && -z ${!NAME_PROC:-} && "0$GEN_TOPO_AUTOSCALE_PROCESSES" == "01" && ($WORKFLOWMODE != "print" || $GEN_TOPO_RUN_HOME_TEST == 1) && $4 != 0 ]]; then + if [[ ${GEN_TOPO_AUTOSCALE_PROCESSES_GLOBAL_WORKFLOW:-} == 1 && -z ${!NAME_PROC:-} && ${GEN_TOPO_AUTOSCALE_PROCESSES:-} == 1 && ($WORKFLOWMODE != "print" || ${GEN_TOPO_RUN_HOME_TEST:-} == 1) && $4 != 0 ]]; then echo $1:\$\(\(\($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) \< 16 ? \($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) : 16\)\) else echo $1:$MULT From 37b85c1e00f9db95bbc1b0500f2933ebef3586ec Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 27 Aug 2023 14:48:20 +0200 Subject: [PATCH 0053/1239] Fix more undefined variables --- DATA/common/gen_topo_helper_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index e63313b6f..2964db60a 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -177,7 +177,7 @@ get_N() # USAGE: get_N [processor-name] [DETECTOR_NAME] [RAW|CTF|REST] [threads, local NAME_DEFAULT="N_${5:-}" local MULT=${!NAME_PROC:-$((${!NAME_FACTOR} * ${!NAME_DET:-1} * ${!NAME_PROC_FACTOR:-1} * ${!NAME_DEFAULT:-1}))} [[ ! -z ${EPN_GLOBAL_SCALING:-} && $1 != "gpu-reconstruction" ]] && MULT=$(($MULT * $EPN_GLOBAL_SCALING)) - if [[ ${GEN_TOPO_AUTOSCALE_PROCESSES_GLOBAL_WORKFLOW:-} == 1 && -z ${!NAME_PROC:-} && ${GEN_TOPO_AUTOSCALE_PROCESSES:-} == 1 && ($WORKFLOWMODE != "print" || ${GEN_TOPO_RUN_HOME_TEST:-} == 1) && $4 != 0 ]]; then + if [[ ${GEN_TOPO_AUTOSCALE_PROCESSES_GLOBAL_WORKFLOW:-} == 1 && -z ${!NAME_PROC:-} && ${GEN_TOPO_AUTOSCALE_PROCESSES:-} == 1 && ($WORKFLOWMODE != "print" || ${GEN_TOPO_RUN_HOME_TEST:-} == 1) && ${4:-} != 0 ]]; then echo $1:\$\(\(\($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) \< 16 ? \($MULT*\$AUTOSCALE_PROCESS_FACTOR/100\) : 16\)\) else echo $1:$MULT From 4b56272baba868ddc318b27ba34d3c77765d68fa Mon Sep 17 00:00:00 2001 From: iravasen Date: Tue, 29 Aug 2023 12:16:53 +0200 Subject: [PATCH 0054/1239] added add_W for all ITS calib workflows --- DATA/production/calib/its-noise-aggregator.sh | 18 +++++++++++++----- DATA/production/calib/its-noise-processing.sh | 7 ++++--- .../calib/its-threshold-aggregator.sh | 11 ++++++----- .../calib/its-threshold-processing.sh | 9 +++++---- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/DATA/production/calib/its-noise-aggregator.sh b/DATA/production/calib/its-noise-aggregator.sh index b7e3b29aa..d0636f0d9 100755 --- a/DATA/production/calib/its-noise-aggregator.sh +++ b/DATA/production/calib/its-noise-aggregator.sh @@ -16,11 +16,19 @@ fi if [[ -z $NTHREADS ]] ; then NTHREADS=1; fi -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name its-noise-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-noise-input-proxy,method=bind,type=pull,rateLogging=1,transport=zeromq\" | " -WORKFLOW+="o2-its-noise-calib-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --prob-threshold 1e-6 --cut-ib 1e-2 --nthreads ${NTHREADSACC} --processing-mode 1 --pipeline its-noise-calibrator:${NITSACCPIPELINES} ${INPTYPE} | " -WORKFLOW+="o2-its-noise-calib-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --validity-days 730 --prob-threshold 1e-6 --cut-ib 1e-2 --nthreads ${NTHREADSNORM} --processing-mode 2 ${INPTYPE} | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://alio2-cr1-flp199.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " +CCDBPATH1="http://o2-ccdb.internal" +CCDBPATH2="http://alio2-cr1-flp199.cern.ch:8083" +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + CCDBPATH1="http://ccdb-test.cern.ch:8080" + CCDBPATH2="http://ccdb-test.cern.ch:8080" +fi + +WORKFLOW= +add_W o2-dpl-raw-proxy "--proxy-name its-noise-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-noise-input-proxy,method=bind,type=pull,rateLogging=1,transport=zeromq\"" "" 0 +add_W o2-its-noise-calib-workflow "--prob-threshold 1e-6 --cut-ib 1e-2 --nthreads ${NTHREADSACC} --processing-mode 1 --pipeline its-noise-calibrator:${NITSACCPIPELINES} ${INPTYPE}" +add_W o2-its-noise-calib-workflow "--validity-days 730 --prob-threshold 1e-6 --cut-ib 1e-2 --nthreads ${NTHREADSNORM} --processing-mode 2 ${INPTYPE}" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH1\" --sspec-min 0 --sspec-max 0" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH2\" --sspec-min 1 --sspec-max 1 --name-extention dcs" WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" if [ $WORKFLOWMODE == "print" ]; then diff --git a/DATA/production/calib/its-noise-processing.sh b/DATA/production/calib/its-noise-processing.sh index e23b30b7f..ee83fb3d6 100755 --- a/DATA/production/calib/its-noise-processing.sh +++ b/DATA/production/calib/its-noise-processing.sh @@ -19,9 +19,10 @@ fi [[ -z $NITSDECTHREADS ]] && NITSDECTHREADS=4 [[ -z $NITSDECTPIPELINES ]] && NITSDECTPIPELINES=6 -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1\" | " -WORKFLOW+="o2-itsmft-stf-decoder-workflow ${ARGS_ALL} ${OUTTYPE} --configKeyValues \"$ARGS_ALL_CONFIG\" --nthreads ${NITSDECTHREADS} --pipeline its-stf-decoder:${NITSDECTPIPELINES} | " -WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-noise-input-proxy --channel-config \"name=its-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=1\" | " +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-itsmft-stf-decoder-workflow "${OUTTYPE} --nthreads ${NITSDECTHREADS} --pipeline its-stf-decoder:${NITSDECTPIPELINES}" +add_W o2-dpl-output-proxy "--dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-noise-input-proxy --channel-config \"name=its-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=1\"" "" 0 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then diff --git a/DATA/production/calib/its-threshold-aggregator.sh b/DATA/production/calib/its-threshold-aggregator.sh index 572a9fb79..cc788d79e 100755 --- a/DATA/production/calib/its-threshold-aggregator.sh +++ b/DATA/production/calib/its-threshold-aggregator.sh @@ -22,14 +22,15 @@ if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGI CCDBPATH2="http://ccdb-test.cern.ch:8080" fi -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --exit-transition-timeout 20 --proxy-name its-thr-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-thr-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " -WORKFLOW+="o2-its-threshold-aggregator-workflow -b $ARGS_ALL | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$CCDBPATH1\" --sspec-min 0 --sspec-max 0 --name-extention dcs | " +WORKFLOW= +add_W o2-dpl-raw-proxy "--exit-transition-timeout 20 --proxy-name its-thr-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=its-thr-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\"" "" 0 +add_W o2-its-threshold-aggregator-workflow "-b" "" 0 +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH1\" --sspec-min 0 --sspec-max 0 --name-extention dcs" if [ $RUNTYPE_ITS == "digital" ]; then - WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$CCDBPATH2\" --sspec-min 1 --sspec-max 1 | " + add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDBPATH2\" --sspec-min 1 --sspec-max 1" fi -WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then echo Workflow command: diff --git a/DATA/production/calib/its-threshold-processing.sh b/DATA/production/calib/its-threshold-processing.sh index ce2a04844..8a768f8c1 100755 --- a/DATA/production/calib/its-threshold-processing.sh +++ b/DATA/production/calib/its-threshold-processing.sh @@ -35,16 +35,17 @@ if [ $RUNTYPE_ITS == "totfullfast" ]; then ADDITIONAL_OPTIONS_CAL="--calculate-slope --charge-a 30 --charge-b 60 --ninj 10" fi -WORKFLOW="o2-dpl-raw-proxy --exit-transition-timeout 20 $ARGS_ALL --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=0,transport=shmem\" | " -WORKFLOW+="o2-itsmft-stf-decoder-workflow ${ARGS_ALL} ${ADDITIONAL_OPTIONS_DEC} --condition-tf-per-query -1 --condition-backend \"http://localhost:8084\" --ignore-dist-stf --configKeyValues \"$ARGS_ALL_CONFIG\" --nthreads 1 --no-clusters --no-cluster-patterns --pipeline its-stf-decoder:${NDECODERS} --enable-calib-data --digits | " +WORKFLOW= +add_W o2-dpl-raw-proxy "--exit-transition-timeout 20 --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=0,transport=shmem\"" "" 0 +add_W o2-itsmft-stf-decoder-workflow "${ADDITIONAL_OPTIONS_DEC} --condition-tf-per-query -1 --condition-backend \"http://localhost:8084\" --ignore-dist-stf --nthreads 1 --no-clusters --no-cluster-patterns --pipeline its-stf-decoder:${NDECODERS} --enable-calib-data --digits" for i in $(seq 0 $((CHIPMODBASE-1))) do - WORKFLOW+="o2-its-threshold-calib-workflow -b ${ADDITIONAL_OPTIONS_CAL} --enable-single-pix-tag --ccdb-mgr-url=\"http://localhost:8084\" --nthreads 1 --chip-mod-selector $i --chip-mod-base $CHIPMODBASE --fittype derivative --output-dir \"/data/calibration\" --meta-output-dir \"/data/epn2eos_tool/epn2eos\" --meta-type \"calibration\" $ARGS_ALL | " + add_W o2-its-threshold-calib-workflow "-b ${ADDITIONAL_OPTIONS_CAL} --enable-single-pix-tag --ccdb-mgr-url=\"http://localhost:8084\" --nthreads 1 --chip-mod-selector $i --chip-mod-base $CHIPMODBASE --fittype derivative --output-dir \"/data/calibration\" --meta-output-dir \"/data/epn2eos_tool/epn2eos\" --meta-type \"calibration\"" "" 0 done if workflow_has_parameter QC && has_detector_qc ITS; then add_QC_from_consul "/o2/components/qc/ANY/any/its-qc-calibration" "--local --host epn -b" fi -WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-thr-input-proxy --channel-config \"name=its-thr-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " +add_W o2-dpl-output-proxy "--dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-thr-input-proxy --channel-config \"name=its-thr-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\"" "" 0 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then From 33fe3316b3080585d288b4101e3903245a0e971f Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Tue, 29 Aug 2023 13:06:27 +0200 Subject: [PATCH 0055/1239] TPC - cleanup of workflows (#1198) * Change of laser workflows for Dense packade format * cleanup Pulser workflow * cleanup tpc-pulser-long * cleanup tpc-pedestal * cleanup tpc-laser * cleanup tpc-laser-filter and tpc-laser-aggregator * Remove hardwired parameters * Fixes for workflow * Add laser raw-filter workflow and move pattern_generator workflow to TPC/test * laser raw-filter cleanup * fix worng character * Bugfix * add globalopt * Add gen_topo_helper_functions --- DATA/production/calib/tpc-laser-aggregator.sh | 52 +++++++++---------- DATA/production/calib/tpc-laser-filter.sh | 21 ++------ DATA/production/calib/tpc-laser.sh | 47 ++++++----------- DATA/production/calib/tpc-pedestal.sh | 39 ++++++++------ DATA/production/calib/tpc-pulser-long.sh | 46 ++++++++++------ DATA/production/calib/tpc-pulser.sh | 41 ++++++++------- DATA/production/standalone-calibration.desc | 4 +- .../detectors/TPC/tpc-laser-raw-filter.sh | 4 +- .../detectors/TPC/tpc-pattern-generator.sh | 6 ++- 9 files changed, 128 insertions(+), 132 deletions(-) diff --git a/DATA/production/calib/tpc-laser-aggregator.sh b/DATA/production/calib/tpc-laser-aggregator.sh index 8ffde6f29..5ad585363 100755 --- a/DATA/production/calib/tpc-laser-aggregator.sh +++ b/DATA/production/calib/tpc-laser-aggregator.sh @@ -2,16 +2,22 @@ source common/setenv.sh -# --------------------------------------------------------------------------------------------------------------------- -# Set general arguments source common/getCommonArgs.sh PROXY_INSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSREFS" -max_events=200 -publish_after=430 +CALIB_CONFIG= "TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" \ + +CCDB_PATH="http://o2-ccdb.internal" + +HOST=localhost + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + +max_events=300 +publish_after=440 min_tracks=0 -num_lanes=1 +num_lanes=36 if [[ ! -z ${TPC_CALIB_MAX_EVENTS:-} ]]; then max_events=${TPC_CALIB_MAX_EVENTS} @@ -27,30 +33,20 @@ if [[ ! -z ${TPC_CALIB_LANES_PAD_RAW:-} ]]; then num_lanes=${TPC_CALIB_LANES_PAD_RAW} fi +EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes #{num_lanes} --check-calib-infos" + -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL \ --proxy-name tpc-laser-input-proxy \ - --dataspec \"$PROXY_INSPEC\" \ + --dataspec "${PROXY_INSPEC}" \ --network-interface ib0 \ - --channel-config \"name=tpc-laser-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" \ - | o2-tpc-calib-laser-tracks $ARGS_ALL \ + --channel-config "name=tpc-laser-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq" \ + | o2-tpc-calib-laser-tracks ${ARGS_ALL} \ --use-filtered-tracks --only-publish-on-eos --min-tfs=${min_tracks}\ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --configKeyValues \"TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null\" \ - --lanes ${num_lanes} \ - --publish-after-tfs ${publish_after} \ - --max-events ${max_events} - --calib-type ce \ - --check-calib-infos \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - --ccdb-path http://o2-ccdb.internal \ - | o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" - -if [ $WORKFLOWMODE == "print" ]; then - echo Workflow command: - echo $WORKFLOW | sed "s/| */|\n/g" -else - # Execute the command we have assembled - WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" - eval $WORKFLOW -fi + | o2-tpc-calib-pad-raw ${ARGS_ALL} \ + --configKeyValues ${CALIB_CONFIG} \ + ${EXTRA_CONFIG} \ + | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ + --ccdb-path ${CCDB_PATH} \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 7ffe4a446..2c4344e54 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -1,16 +1,15 @@ #!/usr/bin/env bash source common/setenv.sh -export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM -export GPUMEMSIZE=$(( 24 << 30 )) -export HOSTMEMSIZE=$(( 5 << 30 )) -export GPUTYPE="HIP" + +source common/getCommonArgs.sh + +source common/gen_topo_helper_functions.sh FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" FILEWORKDIR2="/home/epn/odc/files/" -source common/getCommonArgs.sh ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" if [ $NUMAGPUIDS != 0 ]; then @@ -26,7 +25,6 @@ if [ $GPUTYPE == "HIP" ]; then GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;" GPU_CONFIG+=" --environment ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}" export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 else GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" fi @@ -41,16 +39,10 @@ if [ $HOSTMEMSIZE != "0" ]; then GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" fi -#source /home/epn/runcontrol/tpc/qc_test_env.sh > /dev/null PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSREFS" -### Comment: MAKE SURE the channels match address=ipc://@tf-builder-pipe-0 - -#VERBOSE="" - -#echo GPU_CONFIG $GPU_CONFIG_KEYS; LASER_DECODER_ADD='' @@ -82,8 +74,5 @@ o2-dpl-raw-proxy $ARGS_ALL \ --proxy-name tpc-laser-input-proxy \ --proxy-channel-name tpc-laser-input-proxy \ --channel-config "name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0" \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} - -# --pipeline tpc-tracker:4 \ + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} -# --configKeyValues "align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;GPU_proc.deviceNum=0;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$ARGS_FILES;keyval.output_dir=/dev/null" \ diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index a05854df3..8b1b33050 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -1,16 +1,15 @@ #!/usr/bin/env bash source common/setenv.sh -export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM -export GPUMEMSIZE=$(( 24 << 30 )) -export HOSTMEMSIZE=$(( 5 << 30 )) -export GPUTYPE="HIP" + +source common/getCommonArgs.sh + +source common/gen_topo_helper_functions.sh FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" FILEWORKDIR2="/home/epn/odc/files/" -source common/getCommonArgs.sh ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" if [ $NUMAGPUIDS != 0 ]; then @@ -26,7 +25,6 @@ if [ $GPUTYPE == "HIP" ]; then GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;" GPU_CONFIG+=" --environment ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}" export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 else GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" fi @@ -41,21 +39,15 @@ if [ $HOSTMEMSIZE != "0" ]; then GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" fi -#source /home/epn/runcontrol/tpc/qc_test_env.sh > /dev/null PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -### Comment: MAKE SURE the channels match address=ipc://@tf-builder-pipe-0 - -if [ -z $TPC_LASER_EVENTS ]; then - TPC_LASER_EVENTS=10 -fi +CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" -#VERBOSE="" - -#echo GPU_CONFIG $GPU_CONFIG_KEYS; +CCDB_PATH="http://o2-ccdb.internal" HOST=localhost + QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" max_events=300 @@ -77,16 +69,14 @@ if [[ ! -z ${TPC_CALIB_LANES_PAD_RAW:-} ]]; then num_lanes=${TPC_CALIB_LANES_PAD_RAW} fi +EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes ${num_lanes} --check-calib-infos" LASER_DECODER_ADD='' if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then - LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" + LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root --decoder-type 0" fi - - - o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ @@ -103,21 +93,14 @@ o2-dpl-raw-proxy $ARGS_ALL \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ $GPU_CONFIG \ --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField" \ - --configKeyValues "$ARGS_ALL_CONFIG;align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1" \ + --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1" \ | o2-tpc-laser-track-filter $ARGS_ALL \ | o2-tpc-calib-laser-tracks $ARGS_ALL --use-filtered-tracks --only-publish-on-eos --min-tfs=${min_tracks}\ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --configKeyValues "TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" \ - --lanes ${num_lanes} \ - --calib-type ce \ - --publish-after-tfs ${publish_after} \ - --max-events ${max_events} \ - --check-calib-infos \ + | o2-tpc-calib-pad-raw ${ARGS_ALL} \ + --configKeyValues ${CALIB_CONFIG} ${EXTRA_CONFIG} \ | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - --ccdb-path http://o2-ccdb.internal \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + --ccdb-path ${CCDB_PATH} \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ + | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} -# --pipeline tpc-tracker:4 \ -# --configKeyValues "align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;GPU_proc.deviceNum=0;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$ARGS_FILES;keyval.output_dir=/dev/null" \ diff --git a/DATA/production/calib/tpc-pedestal.sh b/DATA/production/calib/tpc-pedestal.sh index 95280167a..b60bcf28c 100755 --- a/DATA/production/calib/tpc-pedestal.sh +++ b/DATA/production/calib/tpc-pedestal.sh @@ -4,6 +4,8 @@ source common/setenv.sh source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh + if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi @@ -11,6 +13,14 @@ fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +CALIB_CONFIG="TPCCalibPedestal.LastTimeBin=12000;keyval.output_dir=/dev/null" + +CCDB_PATH="http://o2-ccdb.internal" + +HOST=localhost + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + max_events=50 publish_after=400 @@ -22,22 +32,19 @@ if [[ ! -z ${TPC_CALIB_PUBLISH_AFTER:-} ]]; then publish_after=${TPC_CALIB_PUBLISH_AFTER} fi - -CALIB_CONFIG="TPCCalibPedestal.LastTimeBin=12000" -EXTRA_CONFIG=" " -CCDB_PATH="--ccdb-path http://o2-ccdb.internal" EXTRA_CONFIG=" --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36" -HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + +################################################################################################################################# + +o2-dpl-raw-proxy ${ARGS_ALL} \ + --dataspec "${PROXY_INSPEC}" \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$CALIB_CONFIG;keyval.output_dir=/dev/null" \ - $EXTRA_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - $CCDB_PATH \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + | o2-tpc-calib-pad-raw ${ARGS_ALL} \ + --input-spec "${CALIB_INSPEC}" \ + --configKeyValues "${CALIB_CONFIG}" \ + ${EXTRA_CONFIG} \ + | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ + --ccdb-path ${CCDB_PATH} \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ + | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/production/calib/tpc-pulser-long.sh b/DATA/production/calib/tpc-pulser-long.sh index 683da9a55..91c9959b9 100755 --- a/DATA/production/calib/tpc-pulser-long.sh +++ b/DATA/production/calib/tpc-pulser-long.sh @@ -12,24 +12,38 @@ fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=260;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=125;TPCCalibPulser.XmaxT0=145;TPCCalibPulser.NbinsT0=800" +CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=260;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=125;TPCCalibPulser.XmaxT0=145;TPCCalibPulser.NbinsT0=800;keyval.output_dir=/dev/null" -EXTRA_CONFIG=" " -EXTRA_CONFIG="--calib-type pulser --reset-after-publish --publish-after-tfs 200 --max-events 1000000 --lanes 36 --check-calib-infos" -#EXTRA_CONFIG="--calib-type pulser --publish-after-tfs 120 --max-events 10000 --lanes 36 --check-calib-infos" +CCDB_PATH="http://o2-ccdb.internal" -CCDB_PATH="--ccdb-path http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://aliecs.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + +max_events=1000000 +publish_after=200 + +if [[ ! -z ${TPC_CALIB_MAX_EVENTS:-} ]]; then + max_events=${TPC_CALIB_MAX_EVENTS} +fi + +if [[ ! -z ${TPC_CALIB_PUBLISH_AFTER:-} ]]; then + publish_after=${TPC_CALIB_PUBLISH_AFTER} +fi + +EXTRA_CONFIG="--calib-type pulser --reset-after-publish --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" + + +################################################################################################################################# + +o2-dpl-raw-proxy ${ARGS_ALL} \ + --dataspec ${PROXY_INSPEC} \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$CALIB_CONFIG;keyval.output_dir=/dev/null" \ - $EXTRA_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - $CCDB_PATH \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + | o2-tpc-calib-pad-raw ${ARGS_ALL} \ + --input-spec ${CALIB_INSPEC} \ + --configKeyValues "${CALIB_CONFIG}" \ + ${EXTRA_CONFIG} \ + | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ + --ccdb-path ${CCDB_PATH} \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ + | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index 3bef54265..eb45e3a6a 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -5,6 +5,8 @@ source common/setenv.sh source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh + if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi @@ -12,13 +14,17 @@ fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=260;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=125;TPCCalibPulser.XmaxT0=145;TPCCalibPulser.NbinsT0=800" +CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=260;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=125;TPCCalibPulser.XmaxT0=145;TPCCalibPulser.NbinsT0=800;keyval.output_dir=/dev/null" + +CCDB_PATH="http://o2-ccdb.internal" + +HOST=localhost + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" -CCDB_PATH="--ccdb-path http://ccdb-test.cern.ch:8080" max_events=200 publish_after=230 - if [[ ! -z ${TPC_CALIB_MAX_EVENTS:-} ]]; then max_events=${TPC_CALIB_MAX_EVENTS} fi @@ -27,24 +33,19 @@ if [[ ! -z ${TPC_CALIB_PUBLISH_AFTER:-} ]]; then publish_after=${TPC_CALIB_PUBLISH_AFTER} fi - - - EXTRA_CONFIG="--calib-type pulser --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" -#EXTRA_CONFIG="--calib-type pulser --publish-after-tfs 2030 --max-events 200 --lanes 36 --check-calib-infos" -CCDB_PATH="--ccdb-path http://o2-ccdb.internal" -HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ +################################################################################################################################# + +o2-dpl-raw-proxy ${ARGS_ALL} \ + --dataspec "${PROXY_INSPEC}" \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$CALIB_CONFIG;keyval.output_dir=/dev/null" \ - $EXTRA_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - $CCDB_PATH \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + | o2-tpc-calib-pad-raw ${ARGS_ALL} \ + --input-spec "${CALIB_INSPEC}" \ + --configKeyValues "${CALIB_CONFIG}" \ + ${EXTRA_CONFIG} \ + | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ + --ccdb-path ${CCDB_PATH} \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ + | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index 6cb943d4d..9d4d66b3d 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -32,9 +32,9 @@ TPC-pulser-long: "O2PDPSuite" reco,1,1," production/calib/tpc-pulser-long.sh" TPC-pedestal: "O2PDPSuite" reco,1,1," production/calib/tpc-pedestal.sh" -TPC-laser: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser.sh" +TPC-laser: "O2PDPSuite" reco,1,1,"SHMSIZE=$((128 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser.sh" -TPC-laser-multi: "O2PDPSuite" reco,10,10,"SHMSIZE=$((64 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser-filter.sh" calib,1,"production/calib/tpc-laser-aggregator.sh" +TPC-laser-multi: "O2PDPSuite" reco,10,10,"SHMSIZE=$((128 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) production/calib/tpc-laser-filter.sh" calib,1,"production/calib/tpc-laser-aggregator.sh" MFT-noise-calibration: "O2PDPSuite" reco,20,20,"production/calib/mft-noise-processing.sh" calib,20,"production/calib/mft-noise-aggregator.sh" diff --git a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh index 39a4a3deb..557b25c42 100755 --- a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh +++ b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh @@ -4,6 +4,8 @@ source common/setenv.sh source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh + export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder if [ $NUMAGPUIDS != 0 ]; then @@ -42,4 +44,4 @@ o2-dpl-raw-proxy $ARGS_ALL \ --time-bins-before 20 \ --max-time-bins 650 \ | o2-qc $ARGS_ALL --config ${QC_CONFIG} --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/testing/detectors/TPC/tpc-pattern-generator.sh b/DATA/testing/detectors/TPC/tpc-pattern-generator.sh index 74bf683ce..d6a0550ad 100755 --- a/DATA/testing/detectors/TPC/tpc-pattern-generator.sh +++ b/DATA/testing/detectors/TPC/tpc-pattern-generator.sh @@ -1,12 +1,16 @@ #!/usr/bin/env bash source common/setenv.sh + +source common/getCommonArgs.sh + +source common/gen_topo_helper_functions.sh + export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM export GPUMEMSIZE=$(( 24 << 30 )) export HOSTMEMSIZE=$(( 5 << 30 )) export GPUTYPE="HIP" -source common/getCommonArgs.sh if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" From b5b17e39078eabcc3aafe91d7c486f1aca5f43bc Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 29 Aug 2023 18:54:08 +0200 Subject: [PATCH 0056/1239] TRD QC json structure changed (#1195) --- DATA/common/gen_topo_helper_functions.sh | 15 ++ DATA/production/qc-async/trd.json | 45 ++++-- DATA/production/qc-async/trditstpc.json | 76 ---------- DATA/production/qc-sync/trd.json | 96 ++++++++----- DATA/production/qc-sync/trditstpc.json | 175 ----------------------- DATA/production/qc-workflow.sh | 41 +++--- 6 files changed, 129 insertions(+), 319 deletions(-) delete mode 100644 DATA/production/qc-async/trditstpc.json delete mode 100644 DATA/production/qc-sync/trditstpc.json diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index 2964db60a..6a3e84149 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -165,6 +165,21 @@ add_semicolon_separated() done } +add_pipe_separated() +{ + if (( $# < 2 )); then + echo "$# parameters received" + echo "Function name: ${FUNCNAME} expects at least 2 parameters:" + echo "it concatenates the string in 1st parameter by the following" + echo "ones, forming pipe-separated string. $# parameters received" + exit 1 + fi + + for ((i = 2; i <= $#; i++ )); do + eval $1+="\|${!i}" + done +} + # --------------------------------------------------------------------------------------------------------------------- # Helper functions for multiplicities diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index a5c3136b2..65db89ddb 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -23,9 +23,8 @@ } }, "tasks": { - "TRDDigits": { + "Digits": { "active": "true", - "taskName": "Digits", "className": "o2::quality_control_modules::trd::DigitsTask", "moduleName": "QcTRD", "detectorName": "TRD", @@ -33,18 +32,11 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0" + "query": "digits:TRD/DIGITS;triggers:TRD/TRKTRGRD" } }, - "TRDTracklets": { + "Tracklets": { "active": "true", - "taskName": "Tracklets", "className": "o2::quality_control_modules::trd::TrackletsTask", "moduleName": "QcTRD", "detectorName": "TRD", @@ -52,7 +44,35 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" + "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" + } + }, + "PHTrackMatch": { + "active": "true", + "className": "o2::quality_control_modules::trd::PulseHeightTrackMatch", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "phValues:TRD/PULSEHEIGHT" + } + }, + "Tracking": { + "active": "false", + "className": "o2::quality_control_modules::trd::TrackingTask", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC" + }, + "taskParameters": { + "detailedQC": "false", + "trackSources": "ITS-TPC-TRD" } } } @@ -60,4 +80,3 @@ "dataSamplingPolicies": [ ] } - diff --git a/DATA/production/qc-async/trditstpc.json b/DATA/production/qc-async/trditstpc.json deleted file mode 100644 index 44b82903b..000000000 --- a/DATA/production/qc-async/trditstpc.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "type": "2", - "number": "42" - }, - "monitoring": { - "url": "influxdb-unix:///tmp/telegraf.sock" - }, - "consul": { - "url": "alio2-cr1-hv-aliecs:8500" - }, - "conditionDB": { - "url": "alio2-cr1-hv-qcdb1.cern.ch:8083" - } - }, - "tasks": { - "TRDDigits": { - "active": "true", - "taskName": "Digits", - "className": "o2::quality_control_modules::trd::DigitsTask", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0" - } - }, - "TRDTracklets": { - "active": "true", - "taskName": "Tracklets", - "className": "o2::quality_control_modules::trd::TrackletsTask", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" - } - }, - "TRDPulseHeightTrackMatch": { - "active": "true", - "taskName": "PulseHeightTrackMatch", - "className": "o2::quality_control_modules::trd::PulseHeightTrackMatch", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;tracks:TRD/MATCH_ITSTPC;trigrectrk:TRD/TRGREC_ITSTPC;phValues:TRD/PULSEHEIGHT" - } - } - } - }, - "dataSamplingPolicies": [ - ] -} - diff --git a/DATA/production/qc-sync/trd.json b/DATA/production/qc-sync/trd.json index d51527f13..9e14e542b 100644 --- a/DATA/production/qc-sync/trd.json +++ b/DATA/production/qc-sync/trd.json @@ -28,23 +28,16 @@ } }, "tasks": { - "TRDRawData": { + "RawData": { "active": "true", - "taskName": "RawData", "className": "o2::quality_control_modules::trd::RawData", "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", "maxNumberCycles": "-1", "dataSource": { - "type": "dataSamplingPolicy", - "name": "trdall" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "0.0", - "pulseheightpeakupper": "5.0" + "type": "direct", + "query": "rawstats:TRD/RAWSTATS" }, "location": "local", "localMachines": [ @@ -52,14 +45,13 @@ "localhost" ], "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47742", + "remotePort": "29850", "mergingMode": "delta", "mergerCycleMultiplier": "2", "localControl": "odc" }, - "TRDDigits": { + "Digits": { "active": "true", - "taskName": "Digits", "className": "o2::quality_control_modules::trd::DigitsTask", "moduleName": "QcTRD", "detectorName": "TRD", @@ -69,20 +61,13 @@ "type": "dataSamplingPolicy", "name": "trdall" }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0", - "ignorelayerlabels": "1" - }, "location": "local", "localMachines": [ "epn", "localhost" ], "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47743", + "remotePort": "29851", "mergingMode": "delta", "mergerCycleMultiplier": "2", "localControl": "odc", @@ -91,9 +76,8 @@ "1" ] }, - "TRDTracklets": { + "Tracklets": { "active": "true", - "taskName": "Tracklets", "className": "o2::quality_control_modules::trd::TrackletsTask", "moduleName": "QcTRD", "detectorName": "TRD", @@ -103,20 +87,13 @@ "type": "dataSamplingPolicy", "name": "trdall" }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0", - "ignorelayerlabels": "1" - }, "location": "local", "localMachines": [ "epn", "localhost" ], "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47744", + "remotePort": "29852", "mergingMode": "delta", "mergerCycleMultiplier": "2", "localControl": "odc", @@ -124,6 +101,61 @@ "3", "1" ] + }, + "PHTrackMatch": { + "active": "true", + "taskName": "PHTrackMatch", + "className": "o2::quality_control_modules::trd::PulseHeightTrackMatch", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "phValues:TRD/PULSEHEIGHT" + }, + "location": "local", + "localMachines": [ + "epn", + "localhost" + ], + "remoteMachine": "alio2-cr1-qc05.cern.ch", + "remotePort": "29853", + "mergingMode": "delta", + "mergerCycleMultiplier": "2", + "mergersPerLayer": [ + "3", + "1" + ] + }, + "Tracking": { + "active": "false", + "className": "o2::quality_control_modules::trd::TrackingTask", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC" + }, + "location": "local", + "localMachines": [ + "epn", + "localhost" + ], + "taskParameters": { + "detailedQC": "false", + "trackSources": "ITS-TPC-TRD" + }, + "remoteMachine": "alio2-cr1-qc05.cern.ch", + "remotePort": "29854", + "mergingMode": "delta", + "mergerCycleMultiplier": "2", + "mergersPerLayer": [ + "3", + "1" + ] } } }, @@ -132,7 +164,7 @@ "id": "trdall", "active": "true", "machines": [], - "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;rawstats:TRD/RAWSTATS/0", + "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0", "samplingConditions": [ { "condition": "random", diff --git a/DATA/production/qc-sync/trditstpc.json b/DATA/production/qc-sync/trditstpc.json deleted file mode 100644 index 6957cfc06..000000000 --- a/DATA/production/qc-sync/trditstpc.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ali-qcdb.cern.ch:8083", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "type": "2", - "number": "42" - }, - "monitoring": { - "url": "influxdb-unix:///tmp/telegraf.sock" - }, - "consul": { - "url": "http://localhost:8500" - }, - "conditionDB": { - "url": "o2-ccdb.internal" - }, - "infologger": { - "": "Message at this level or above are discarded (default: 21 - Trace)", - "filterDiscardDebug": "false", - "filterDiscardLevel": "11" - } - }, - "tasks": { - "TRDRawData": { - "active": "true", - "taskName": "RawData", - "className": "o2::quality_control_modules::trd::RawData", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "dataSamplingPolicy", - "name": "trdall" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "0.0", - "pulseheightpeakupper": "5.0" - }, - "location": "local", - "localMachines": [ - "epn", - "localhost" - ], - "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47742", - "mergingMode": "delta", - "mergerCycleMultiplier": "2", - "localControl": "odc" - }, - "TRDDigits": { - "active": "true", - "taskName": "Digits", - "className": "o2::quality_control_modules::trd::DigitsTask", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "dataSamplingPolicy", - "name": "trdall" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0", - "ignorelayerlabels": "1" - }, - "location": "local", - "localMachines": [ - "epn", - "localhost" - ], - "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47743", - "mergingMode": "delta", - "mergerCycleMultiplier": "2", - "localControl": "odc", - "mergersPerLayer": [ - "3", - "1" - ] - }, - "TRDTracklets": { - "active": "true", - "taskName": "Tracklets", - "className": "o2::quality_control_modules::trd::TrackletsTask", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "dataSamplingPolicy", - "name": "trdall" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0", - "ignorelayerlabels": "1" - }, - "location": "local", - "localMachines": [ - "epn", - "localhost" - ], - "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47744", - "mergingMode": "delta", - "mergerCycleMultiplier": "2", - "localControl": "odc", - "mergersPerLayer": [ - "3", - "1" - ] - }, - "TRDPulseHeightTrackMatch": { - "active": "true", - "taskName": "TRDPulseHeightTrackMatch", - "className": "o2::quality_control_modules::trd::PulseHeightTrackMatch", - "moduleName": "QcTRD", - "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "dataSamplingPolicy", - "name": "trdall" - }, - "taskParameters": { - "peakregionstart": "7.0", - "peakregionend": "20.0", - "pulseheightpeaklower": "1.0", - "pulseheightpeakupper": "5.0" - }, - "location": "local", - "localMachines": [ - "epn", - "localhost" - ], - "remoteMachine": "alio2-cr1-qc05.cern.ch", - "remotePort": "47745", - "mergingMode": "delta", - "mergerCycleMultiplier": "2", - "localControl": "odc" - } - } - }, - "dataSamplingPolicies": [ - { - "id": "trdall", - "active": "true", - "machines": [], - "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;rawstats:TRD/RAWSTATS/0;tracks:TRD/MATCH_ITSTPC;trigrectrk:TRD/TRGREC_ITSTPC;phValues:TRD/PULSEHEIGHT", - "samplingConditions": [ - { - "condition": "random", - "fraction": "0.05", - "seed": "1234" - } - ], - "blocking": "false" - } - ] -} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index dc557861f..158eaa420 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -40,6 +40,7 @@ add_QC_JSON() { QC_CONFIG= QC_CONFIG_OVERRIDE= +: ${QC_DETECTOR_CONFIG_OVERRIDE:=} # set to empty string only if not already set externally if [[ -z ${QC_JSON_FROM_OUTSIDE:-} && ! -z ${GEN_TOPO_QC_JSON_FILE:-} && -f $GEN_TOPO_QC_JSON_FILE ]]; then QC_JSON_FROM_OUTSIDE=$GEN_TOPO_QC_JSON_FILE elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then @@ -64,7 +65,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_FT0:-}" ]] && QC_JSON_FT0=consul://o2/components/qc/ANY/any/ft0-digits-qc-epn [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=consul://o2/components/qc/ANY/any/fv0-digits-qc-epn if [[ -z "${QC_JSON_EMC:-}" ]]; then - if [ "$BEAMTYPE" == "PbPb"]; then + if [[ "$BEAMTYPE" == "PbPb" ]]; then if has_detector CTP; then QC_JSON_EMC=consul://o2/components/qc/ANY/any/emc-qcmn-epnall-withCTP-PbPb else @@ -98,13 +99,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then fi fi [[ -z "${QC_JSON_CPV:-}" ]] && QC_JSON_CPV=consul://o2/components/qc/ANY/any/cpv-physics-qcmn-epn - if [[ -z "${QC_JSON_TRD:-}" ]]; then - if has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRD; then - QC_JSON_TRD=consul://o2/components/qc/ANY/any/trd-full-qcmn-epn - else - QC_JSON_TRD=consul://o2/components/qc/ANY/any/trd-full-qcmn-nopulseheight-epn - fi - fi + [[ -z "${QC_JSON_TRD:-}" ]] && QC_JSON_TRD=consul://o2/components/qc/ANY/any/trd-full-qcmn [[ -z "${QC_JSON_PHS:-}" ]] && QC_JSON_PHS=consul://o2/components/qc/ANY/any/phos-raw-clusters-epn [[ -z "${QC_JSON_GLO_PRIMVTX:-}" ]] && QC_JSON_GLO_PRIMVTX=consul://o2/components/qc/ANY/any/glo-vtx-qcmn-epn [[ -z "${QC_JSON_GLO_ITSTPC:-}" ]] && QC_JSON_GLO_ITSTPC=consul://o2/components/qc/ANY/any/glo-itstpc-mtch-qcmn-epn @@ -136,13 +131,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_MID:-}" ]] && QC_JSON_MID=$O2DPG_ROOT/DATA/production/qc-sync/mid-digits.json && has_processing_step MID_RECO && QC_JSON_MID=$O2DPG_ROOT/DATA/production/qc-sync/mid.json [[ -z "${QC_JSON_CPV:-}" ]] && QC_JSON_CPV=$O2DPG_ROOT/DATA/production/qc-sync/cpv.json [[ -z "${QC_JSON_PHS:-}" ]] && QC_JSON_PHS=$O2DPG_ROOT/DATA/production/qc-sync/phs.json - if [[ -z "${QC_JSON_TRD:-}" ]]; then - if has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRD; then - QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-sync/trditstpc.json - else - QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-sync/trd.json - fi - fi + [[ -z "${QC_JSON_TRD:-}" ]] && QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-sync/trd.json [[ -z "${QC_JSON_GLO_PRIMVTX:-}" ]] && QC_JSON_GLO_PRIMVTX=$O2DPG_ROOT/DATA/production/qc-sync/glo-vtx-qcmn-epn.json [[ -z "${QC_JSON_GLO_ITSTPC:-}" ]] && QC_JSON_GLO_ITSTPC=$O2DPG_ROOT/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json @@ -178,13 +167,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then fi [[ -z "${QC_JSON_CPV:-}" ]] && QC_JSON_CPV=$O2DPG_ROOT/DATA/production/qc-async/cpv.json [[ -z "${QC_JSON_PHS:-}" ]] && QC_JSON_PHS=$O2DPG_ROOT/DATA/production/qc-async/phs.json - if [[ -z "${QC_JSON_TRD:-}" ]]; then - if has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRD; then - QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-async/trditstpc.json - else - QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-async/trd.json - fi - fi + [[ -z "${QC_JSON_TRD:-}" ]] && QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-async/trd.json # the following two ($QC_JSON_PRIMVTX and $QC_JSON_ITSTPC) replace $QC_JSON_GLO for async processing [[ -z "${QC_JSON_GLO_PRIMVTX:-}" ]] && QC_JSON_GLO_PRIMVTX=$O2DPG_ROOT/DATA/production/qc-async/primvtx.json [[ -z "${QC_JSON_GLO_ITSTPC:-}" ]] && QC_JSON_GLO_ITSTPC=$O2DPG_ROOT/DATA/production/qc-async/itstpc.json @@ -216,6 +199,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then add_QC_JSON matchTOF ${QC_JSON_TOF_MATCH} fi + # Detector QC for i in ${LIST_OF_DETECTORS//,/ }; do DET_JSON_FILE="QC_JSON_$i" if has_detector_qc $i && [ ! -z "${!DET_JSON_FILE:-}" ]; then @@ -223,6 +207,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then fi done + # Global reconstruction QC for i in ${LIST_OF_GLORECO//,/ }; do DET_JSON_FILE="QC_JSON_GLO_$i" if has_matching_qc $i && [ ! -z "${!DET_JSON_FILE:-}" ]; then @@ -246,6 +231,16 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then add_QC_JSON EXTRA ${QC_JSON_EXTRA} fi + # extra settings depending on available detectors + # for strings remember to escape e.g. " and ; + # e.g. .qc.tasks.Tracking.taskParameters.dataSource.query=\"tracks:TPC/TRACKS\;clusters:TPC/CLUSTERS\" + if [[ -z "${DISABLE_QC_DETECTOR_CONFIG_OVERRIDE:-}" ]]; then + if ! has_matching_qc ITSTPCTRD || ! has_detectors_reco ITS TPC TRD; then + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.active=false' + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.PHTrackMatch.active=false' + fi + fi + if [[ ! -z "$JSON_FILES" ]]; then if [[ -z "${GEN_TOPO_QC_JSON_FILE:-}" ]]; then mkdir -p $GEN_TOPO_WORKDIR/json_cache @@ -256,7 +251,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then else MERGED_JSON_FILENAME=$GEN_TOPO_QC_JSON_FILE fi - jq -n 'reduce inputs as $s (input; .qc.tasks += ($s.qc.tasks) | .qc.checks += ($s.qc.checks) | .qc.externalTasks += ($s.qc.externalTasks) | .qc.postprocessing += ($s.qc.postprocessing)| .dataSamplingPolicies += ($s.dataSamplingPolicies))' $QC_JSON_GLOBAL $JSON_FILES > $MERGED_JSON_FILENAME + jq -n 'reduce inputs as $s (input; .qc.tasks += ($s.qc.tasks) | .qc.checks += ($s.qc.checks) | .qc.externalTasks += ($s.qc.externalTasks) | .qc.postprocessing += ($s.qc.postprocessing)| .dataSamplingPolicies += ($s.dataSamplingPolicies))'${QC_DETECTOR_CONFIG_OVERRIDE} $QC_JSON_GLOBAL $JSON_FILES > $MERGED_JSON_FILENAME if [[ $? != 0 ]]; then echo Merging QC workflow with JSON files $JSON_FILES failed 1>&2 exit 1 From edb8daab4668195ab0e9a7203161be1e325eb4d7 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 29 Aug 2023 19:00:16 +0200 Subject: [PATCH 0057/1239] [O2-4069] Add dedicated staging QC global config (#1201) --- .../qc-sync/qc-global-epn-staging.json | 36 +++++++++++++++++++ DATA/production/qc-workflow.sh | 6 +++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 DATA/production/qc-sync/qc-global-epn-staging.json diff --git a/DATA/production/qc-sync/qc-global-epn-staging.json b/DATA/production/qc-sync/qc-global-epn-staging.json new file mode 100644 index 000000000..479e52f97 --- /dev/null +++ b/DATA/production/qc-sync/qc-global-epn-staging.json @@ -0,0 +1,36 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "alio2-cr1-hv-mvs00:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2" + }, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "http://127.0.0.1:8084" + }, + "infologger": { + "filterDiscardDebug": "true", + "filterDiscardLevel": "1", + "filterDiscardFile": "../../qc-_ID_.log", + "filterRotateMaxBytes": "100000000", + "filterRotateMaxFiles": "1" + }, + "bookkeeping": { + "url": "" + } + } + } +} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 158eaa420..ff706b329 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -110,7 +110,11 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then QC_JSON_TOF_MATCH=consul://o2/components/qc/ANY/any/tof-qcmn-match-itstpctof fi fi - [[ -z "${QC_JSON_GLOBAL:-}" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global-epn.json # this must be last + if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + [[ -z "${QC_JSON_GLOBAL:-}" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global-epn-staging.json # this must be last + else + [[ -z "${QC_JSON_GLOBAL:-}" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global-epn.json # this must be last + fi elif [[ $SYNCMODE == 1 ]]; then # Sync processing running locally (CI, laptop) [[ -z "${QC_JSON_TPC:-}" ]] && QC_JSON_TPC=$O2DPG_ROOT/DATA/production/qc-sync/tpc.json [[ -z "${QC_JSON_ITS:-}" ]] && QC_JSON_ITS=$O2DPG_ROOT/DATA/production/qc-sync/its.json From 5a965c779b38deedbe0e317a9237417574d4c3b4 Mon Sep 17 00:00:00 2001 From: noferini Date: Wed, 30 Aug 2023 14:06:46 +0200 Subject: [PATCH 0058/1239] adding FT0VX for 5.36 ref energy --- MC/bin/o2dpg_sim_workflow_anchored.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 6157020bc..ad9ba4a0a 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -314,10 +314,12 @@ def main(): effT0 = args.ft0_eff if effT0 < 0: if args.col == "pp": - if args.eCM > 5000: - effT0 = 0.759 - else: + if args.eCM < 1000: effT0 = 0.68 + elif args.eCM < 6000: + effT0 = 0.737 + else: + effT0 = 0.759 elif args.col == "PbPb": effT0 = 4.0 else: From 7f0d165bc887ab22ab8a9aae34811eae7a0d0bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barth=C3=A9l=C3=A9my=20von=20Haller?= Date: Thu, 31 Aug 2023 08:49:04 +0200 Subject: [PATCH 0059/1239] Update qc-global-epn.json (#1205) Set the bookkeeping url --- DATA/production/qc-sync/qc-global-epn.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DATA/production/qc-sync/qc-global-epn.json b/DATA/production/qc-sync/qc-global-epn.json index 36737a084..420bfeeaf 100644 --- a/DATA/production/qc-sync/qc-global-epn.json +++ b/DATA/production/qc-sync/qc-global-epn.json @@ -27,6 +27,9 @@ "filterDiscardFile": "../../qc-_ID_.log", "filterRotateMaxBytes": "100000000", "filterRotateMaxFiles": "1" + }, + "bookkeeping": { + "url": "alio2-cr1-hv-web01.cern.ch:4001" } } } From 6873ad7c61c1be78a83fd28df23c4102b4c9709f Mon Sep 17 00:00:00 2001 From: Antonio FRANCO Date: Thu, 31 Aug 2023 08:52:13 +0200 Subject: [PATCH 0060/1239] Modernization of standalone detector workflows (#1192) Co-authored-by: Antonio Franco --- .../calib/hmp-pedestals-processing.sh | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/DATA/production/calib/hmp-pedestals-processing.sh b/DATA/production/calib/hmp-pedestals-processing.sh index e058114b8..ef6fcdb10 100755 --- a/DATA/production/calib/hmp-pedestals-processing.sh +++ b/DATA/production/calib/hmp-pedestals-processing.sh @@ -2,7 +2,7 @@ # ------------------------------------------------------------------------ # ALICE HMPID detector -# Workflow to calculate pedestals v.1.0 = 4/04/2023 +# Workflow to calculate pedestals v.3.0 = 24/08/2023 # # Extra Env Variables: # HMP_SIGMACUT : The value of sigams for the threshold cut [=4] @@ -15,10 +15,9 @@ # Auth. A.Franco - INFN Sez.BARI - ITALY # ------------------------------------------------------------------------ -source common/setenv.sh -# env > /tmp/hmpid_pedestal_env_dump.txt - # Set general arguments +source common/setenv.sh +source common/gen_topo_helper_functions.sh source common/getCommonArgs.sh # Define the Input/Output streams @@ -53,35 +52,35 @@ then HMP_PED_TAG="Latest" fi -# Here we compose the workflow -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull," -WORKFLOW+="method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\" | " - -WORKFLOW+="o2-hmpid-raw-to-pedestals-workflow ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --fast-decode " - +# Compose the specific parameters +SPEC_PARAM="" if [ $HMP_NODCSCCDB_REC == 'false' ]; then - WORKFLOW+="--use-dcsccdb --dcsccdb-uri 'http://alio2-cr1-flp199.cern.ch:8083' --dcsccdb-alivehours 3 " + SPEC_PARAM+="--use-dcsccdb --dcsccdb-uri 'http://alio2-cr1-flp199.cern.ch:8083' --dcsccdb-alivehours 3 " fi if [ $HMP_CCDB_REC == 'true' ]; then - WORKFLOW+="--use-ccdb --ccdb-uri 'http://o2-ccdb.internal' " + SPEC_PARAM+="--use-ccdb --ccdb-uri 'http://o2-ccdb.internal' " fi if [ $HMP_FILES_REC == 'true' ]; then - WORKFLOW+="--use-files " + SPEC_PARAM+="--use-files " fi -WORKFLOW+="--files-basepath 'HMP' " -WORKFLOW+="--pedestals-tag ${HMP_PED_TAG} --sigmacut ${HMP_SIGMACUT} | " +SPEC_PARAM+="--files-basepath 'HMP' " +SPEC_PARAM+="--pedestals-tag ${HMP_PED_TAG} --sigmacut ${HMP_SIGMACUT}" +# Here we compose the workflow +# Start with an empty workflow +WORKFLOW= +add_W o2-dpl-raw-proxy "$ARGS_ALL --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\"" +add_W o2-hmpid-raw-to-pedestals-workflow "${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --fast-decode $SPEC_PARAM" + +# Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" -if [ $WORKFLOWMODE == "print" ]; then - echo "HMPID Pedestals Calculation (v.0.1) Workflow command:" - echo $WORKFLOW | sed "s/| */|\n/g" -else - # Execute the command we have assembled - WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" - eval $WORKFLOW -fi +[[ $WORKFLOWMODE != "print" ]] && WORKFLOW+=" --${WORKFLOWMODE} ${WORKFLOWMODE_FILE:-}" +[[ $WORKFLOWMODE == "print" || "${PRINT_WORKFLOW:-}" == "1" ]] && echo "#HMPID Pedestals Calculation (v.3) workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2') +if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi + +# ------- EOF -------- From 625a88e25221a7cc22071f1b761d54a473171d07 Mon Sep 17 00:00:00 2001 From: noferini Date: Wed, 30 Aug 2023 16:08:02 +0200 Subject: [PATCH 0061/1239] remove full path for o2-aod-merger --- MC/bin/o2dpg_sim_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 1844b062e..25a939551 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1364,7 +1364,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): AOD_merge_task['cmd'] += ' [ -f input.txt ] && rm input.txt; ' AOD_merge_task['cmd'] += ' [ -f ../AO2D_old.root ] && echo "../AO2D_old.root" > input.txt;' AOD_merge_task['cmd'] += ' echo "./AO2D_repaired.root" >> input.txt;' - AOD_merge_task['cmd'] += ' ${O2PHYSICS_ROOT}/bin/o2-aod-merger --output ../AO2D.root;' + AOD_merge_task['cmd'] += ' o2-aod-merger --output ../AO2D.root;' AOD_merge_task['cmd'] += ' rm ../AO2D_old.root || true' AOD_merge_task['semaphore'] = 'aodmerge' #<---- this is making sure that only one merge is running at any time workflow['stages'].append(AOD_merge_task) @@ -1387,7 +1387,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): AOD_merge_task = createTask(name='aodmerge', needs = aodmergerneeds, lab=["AOD"], mem='2000', cpu='1') AOD_merge_task['cmd'] = ' [ -f aodmerge_input.txt ] && rm aodmerge_input.txt; ' AOD_merge_task['cmd'] += ' for i in `seq 1 ' + str(NTIMEFRAMES) + '`; do echo "tf${i}/AO2D.root" >> aodmerge_input.txt; done; ' -AOD_merge_task['cmd'] += ' ${O2PHYSICS_ROOT}/bin/o2-aod-merger --input aodmerge_input.txt --output AO2D.root' +AOD_merge_task['cmd'] += ' o2-aod-merger --input aodmerge_input.txt --output AO2D.root' workflow['stages'].append(AOD_merge_task) job_merging = False From 4fc5b44161d0aedc3c4be62324d44311ee652b93 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 31 Aug 2023 14:55:25 +0200 Subject: [PATCH 0062/1239] Bugfix: only add extra options if detector is included in the run (#1206) --- DATA/production/qc-workflow.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index ff706b329..a3853b988 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -239,9 +239,11 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then # for strings remember to escape e.g. " and ; # e.g. .qc.tasks.Tracking.taskParameters.dataSource.query=\"tracks:TPC/TRACKS\;clusters:TPC/CLUSTERS\" if [[ -z "${DISABLE_QC_DETECTOR_CONFIG_OVERRIDE:-}" ]]; then - if ! has_matching_qc ITSTPCTRD || ! has_detectors_reco ITS TPC TRD; then - add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.active=false' - add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.PHTrackMatch.active=false' + if has_detector_qc TRD && [[ ! -z ${QC_JSON_TRD:-} ]]; then # extra settings for TRD QC + if ! has_matching_qc ITSTPCTRD || ! has_detectors_reco ITS TPC TRD; then + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.active=false' + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.PHTrackMatch.active=false' + fi fi fi From 46843e5ace87c66b93f07a9fa950e985eca7a702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubo=C5=A1=20Kr=C4=8D=C3=A1l?= Date: Thu, 31 Aug 2023 19:59:59 +0200 Subject: [PATCH 0063/1239] Set default topo merger commands to use installed Python package (#1204) --- DATA/tools/epn/gen_topo.sh | 12 ++++++------ DATA/tools/epn/gen_topo_o2dpg.sh | 3 ++- DATA/tools/parse | 10 +++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 35e3c571e..5d2fa25a9 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -26,17 +26,17 @@ fi [[ -z "$GEN_TOPO_EPN_CCDB_SERVER" ]] && export GEN_TOPO_EPN_CCDB_SERVER="http://127.0.0.1:8084" # CCDB server to use if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi -# Command for topology generation +# Command for topology merging if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /home/lkrcal/epn/topogen/topo_merger.py' + export GEN_TOPO_ODC_EPN_TOPO_CMD='/usr/local/bin/epn-topo-merger' fi + # Command for postprocessing of topology generation after topology caching -export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='/home/lkrcal/epn/topogen/.venv/bin/python3 /home/lkrcal/epn/topogen/topo_resource_alloc.py' +if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then + export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='/usr/local/bin/epn-topo-alloc' +fi # Extra arguments for topology generation -#if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_ARGS" ]]; then -# GEN_TOPO_ODC_EPN_TOPO_ARGS= -#fi if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" ]]; then if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index e9aa5a03c..914470048 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -115,7 +115,8 @@ if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then fi TMP_POST_CACHING_CMD+=" -o $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml" echo "Running post-caching topo-merger command: $TMP_POST_CACHING_CMD" 1>&2 - eval $TMP_POST_CACHING_CMD 1>&2 || { echo Error during EPN topology-merger resource allocation 1>&2; exit 1; } + # Run wih a custom PYTHONPATH in order to import the python packages needed for the topology merger + eval PYTHONPATH="${PYTHONPATH}:/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9" $TMP_POST_CACHING_CMD 1>&2 || { echo Error during EPN topology-merger resource allocation 1>&2; exit 1; } mv -f $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml 1>&2 fi diff --git a/DATA/tools/parse b/DATA/tools/parse index 726991fe7..1f8d82156 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -214,10 +214,7 @@ for line in f: if 'RECO_MAX_FAIL_NODES_OVERRIDE' in os.environ and os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'] != '': reconodesmin = max(1, reconodes - int(os.environ['RECO_MAX_FAIL_NODES_OVERRIDE'])) if os.environ['WORKFLOWMODE'] == 'dds': - if 'GEN_TOPO_ODC_EPN_TOPO_CMD' in os.environ: - odccommand = os.environ['GEN_TOPO_ODC_EPN_TOPO_CMD'] - else: - odccommand = 'odc-epn-topo' + odccommand = os.environ['GEN_TOPO_ODC_EPN_TOPO_CMD'] if 'GEN_TOPO_ODC_EPN_TOPO_ARGS' in os.environ: odccommand += ' ' + os.environ['GEN_TOPO_ODC_EPN_TOPO_ARGS'] if True: @@ -254,9 +251,12 @@ for line in f: odccommand += f" --assets {os.environ['GEN_TOPO_QC_JSON_FILE']}" odccommand += ' -o ' + sys.argv[3] print('Running topology merger command', odccommand) + # Add a custom PYTHONPATH in order to import the python packages needed for the topology merger + epn_topo_env = os.environ.copy() + epn_topo_env["PYTHONPATH"] = f"/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9/site-packages:{epn_topo_env['PYTHONPATH']}" starttime = time.time() try: - out = subprocess.check_output(odccommand, stderr=subprocess.STDOUT, shell=True) + out = subprocess.check_output(odccommand, stderr=subprocess.STDOUT, shell=True, env=epn_topo_env) print(out.decode('utf-8')) print('Topology merger finished successfully') except subprocess.CalledProcessError as e: From 4dc0119d7c436d777658b25359ac94964be623fa Mon Sep 17 00:00:00 2001 From: shahoian Date: Sun, 3 Sep 2023 21:58:39 +0200 Subject: [PATCH 0064/1239] Prescale ITS in pbpb with multiplicity only, no random cut At the moment in the online PbPb the ITS reconstructs by defuilt 30-2000 tracks multiplicity range, corresponding to 53% of collisions and 28% of tracks of LHC22s spectrum. Additionally, we randomly downscale multiplicity-selected ROFs 5%, going to 2.6% collisions and naively 1.4% of all tracks but in reality more since we prioritize ROFs with triggers. O2 https://github.com/AliceO2Group/AliceO2/pull/11853 makes more precise the mult estimated (tuned on LHC22s) and this PR sets the selected ITS multiplicity to 100:200 tracks range, which corresponds to ~7% of LHC22s ITS mult.spectrum and 1% of tracks. The random selection is turned off by default. The same settings are in the defaults of dpl-workflow.sh. If needed, the prescaling can be modified via CUT_MULT_MIN_ITS and CUT_MULT_MAX_ITS env.vars. --- DATA/production/workflow-multiplicities.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 56b1d66fc..f71327797 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -209,6 +209,11 @@ fi if [[ "$HIGH_RATE_PP" == "1" ]]; then : ${CUT_RANDOM_FRACTION_ITS:=0.97} +elif [[ $BEAMTYPE == "PbPb" ]]; then + : ${CUT_RANDOM_FRACTION_ITS:=-1} + : ${CUT_MULT_MIN_ITS:=100} + : ${CUT_MULT_MAX_ITS:=200} + : ${CUT_MULT_VTX_ITS:=20} else : ${CUT_RANDOM_FRACTION_ITS:=0.95} fi From a0cb6f784f257b1b9dc6fcd5feb753b79aebcd45 Mon Sep 17 00:00:00 2001 From: Francesco Noferini Date: Mon, 4 Sep 2023 09:38:04 +0200 Subject: [PATCH 0065/1239] modernization TOF workflows (#1210) --- DATA/production/calib/tof-diagn-aggregator.sh | 19 ++++++++++++++----- .../tof-standalone-cosmic-reco-time-calib.sh | 11 ++++++----- DATA/production/calib/tof-standalone-reco.sh | 11 ++++++----- .../calib/tof-time-calib-aggregator.sh | 18 +++++++++++++----- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/DATA/production/calib/tof-diagn-aggregator.sh b/DATA/production/calib/tof-diagn-aggregator.sh index 25c0e6dca..5cfb7861a 100755 --- a/DATA/production/calib/tof-diagn-aggregator.sh +++ b/DATA/production/calib/tof-diagn-aggregator.sh @@ -5,14 +5,23 @@ source common/setenv.sh # --------------------------------------------------------------------------------------------------------------------- # Set general arguments source common/getCommonArgs.sh -ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;$ALL_EXTRA_CONFIG" +source common/gen_topo_helper_functions.sh PROXY_INSPEC="diagWords:TOF/DIAFREQ/0;eos:***/INFORMATION" -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name tof-diagn-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=tof-diagn-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " -WORKFLOW+="o2-calibration-tof-diagnostic-workflow --tf-per-slot 25000 --max-delay 1 $ARGS_ALL | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ccdb-test.cern.ch:8080\" | " -WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" +WORKFLOW= +add_W o2-dpl-raw-proxy "$ARGS_ALL --proxy-name tof-diagn-input-proxy --dataspec ${PROXY_INSPEC} --network-interface ib0 --channel-config \"name=tof-diagn-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" " +add_W o2-calibration-tof-diagnostic-workflow "--tf-per-slot 25000 --max-delay 1 ${ARGS_ALL}" + +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + CCDB_POPULATOR_UPLOAD_PATH="http://ccdb-test.cern.ch:8080" +else + CCDB_POPULATOR_UPLOAD_PATH="http://localhost:8084" +fi +add_W o2-calibration-ccdb-populator-workflow "${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --ccdb-path=${CCDB_POPULATOR_UPLOAD_PATH} " + + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then echo Workflow command: diff --git a/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh b/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh index 0287cdb86..04d35cc29 100755 --- a/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh +++ b/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh @@ -5,17 +5,18 @@ source common/setenv.sh # --------------------------------------------------------------------------------------------------------------------- # Set general arguments source common/getCommonArgs.sh -ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;$ALL_EXTRA_CONFIG" +source common/gen_topo_helper_functions.sh PROXY_INSPEC="x:TOF/CRAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" NTHREADS=1 PROXY_OUTSPEC="calclus:TOF/INFOCALCLUS;cosmics:TOF/INFOCOSMICS;trkcos:TOF/INFOTRACKCOS;trksiz:TOF/INFOTRACKSIZE" MYDIR="$(dirname $(readlink -f $0))/../../testing/detectors/TOF" -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " -WORKFLOW+="o2-tof-reco-workflow --input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" | " -WORKFLOW+="o2-qc ${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn | " -WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name tof-time-calib-input-proxy --channel-config \"name=tof-time-calib-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " +WORKFLOW= +add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " +add_W o2-tof-reco-workflow "--input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" " +add_W o2-qc "${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn " +add_W o2-dpl-output-proxy "${ARGS_ALL} --dataspec ${PROXY_OUTSPEC} --proxy-channel-name tof-time-calib-input-proxy --channel-config \"name=tof-time-calib-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then diff --git a/DATA/production/calib/tof-standalone-reco.sh b/DATA/production/calib/tof-standalone-reco.sh index a628aa775..bd14b936e 100755 --- a/DATA/production/calib/tof-standalone-reco.sh +++ b/DATA/production/calib/tof-standalone-reco.sh @@ -5,17 +5,18 @@ source common/setenv.sh # --------------------------------------------------------------------------------------------------------------------- # Set general arguments source common/getCommonArgs.sh -ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;$ALL_EXTRA_CONFIG" +source common/gen_topo_helper_functions.sh PROXY_INSPEC="x:TOF/CRAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" NTHREADS=1 PROXY_OUTSPEC="diagWords:TOF/DIAFREQ" MYDIR="$(dirname $(readlink -f $0))/../../testing/detectors/TOF" -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " -WORKFLOW+="o2-tof-reco-workflow --input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" | " -WORKFLOW+="o2-qc ${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn | " -WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name tof-diagn-input-proxy --channel-config \"name=tof-diagn-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " +WORKFLOW= +add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " +add_W o2-tof-reco-workflow "--input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" " +add_W o2-qc "${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn " +add_W o2-dpl-output-proxy "${ARGS_ALL} --dataspec ${PROXY_OUTSPEC} --proxy-channel-name tof-diagn-input-proxy --channel-config \"name=tof-diagn-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then diff --git a/DATA/production/calib/tof-time-calib-aggregator.sh b/DATA/production/calib/tof-time-calib-aggregator.sh index 438e4a270..a76e0c909 100755 --- a/DATA/production/calib/tof-time-calib-aggregator.sh +++ b/DATA/production/calib/tof-time-calib-aggregator.sh @@ -5,14 +5,22 @@ source common/setenv.sh # --------------------------------------------------------------------------------------------------------------------- # Set general arguments source common/getCommonArgs.sh -ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;$ALL_EXTRA_CONFIG" +source common/gen_topo_helper_functions.sh PROXY_INSPEC="calclus:TOF/INFOCALCLUS/0;cosmics:TOF/INFOCOSMICS/0;trkcos:TOF/INFOTRACKCOS/0;trksiz:TOF/INFOTRACKSIZE/0;eos:***/INFORMATION" -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name tof-time-calib-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=tof-time-calib-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " -WORKFLOW+="o2-calibration-tof-calib-workflow --cosmics --do-channel-offset --min-entries 1000 $ARGS_ALL | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ccdb-test.cern.ch:8080\" | " -WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" +WORKFLOW= +add_W o2-dpl-raw-proxy "$ARGS_ALL --proxy-name tof-time-calib-input-proxy --dataspec ${PROXY_INSPEC} --network-interface ib0 --channel-config \"name=tof-time-calib-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" " +add_W o2-calibration-tof-calib-workflow "--cosmics --do-channel-offset --min-entries 1000 ${ARGS_ALL} " + +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + CCDB_POPULATOR_UPLOAD_PATH="http://ccdb-test.cern.ch:8080" +else + CCDB_POPULATOR_UPLOAD_PATH="http://localhost:8084" +fi +add_W o2-calibration-ccdb-populator-workflow "${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --ccdb-path=${CCDB_POPULATOR_UPLOAD_PATH} " + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then echo Workflow command: From 4d249d23e881930f088af0eb8d79aa3f2d5bd5ab Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 Sep 2023 08:50:04 +0200 Subject: [PATCH 0066/1239] Add option to distinguish between triggered and cont TPC data in global workflow --- DATA/common/setenv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 66286b86a..961bc79e4 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -93,6 +93,7 @@ if [[ -z "${EPNSYNCMODE:-}" ]]; then export EPNSYNCMODE=0; fi # if [[ -z "${BEAMTYPE:-}" ]]; then export BEAMTYPE=PbPb; fi # Beam type, must be PbPb, pp, pPb, cosmic, technical if [[ -z "${RUNTYPE:-}" ]]; then export RUNTYPE=Standalone; fi # Run Type, standalone for local tests, otherwise PHYSICS, COSMICS, TECHNICAL, SYNTHETIC if [[ -z "${IS_SIMULATED_DATA:-}" ]]; then export IS_SIMULATED_DATA=1; fi # processing simulated data +if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=1; fi # processing triggered data (TPC triggered instead of continuous) if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing From 2642aebd9103fc8891f282f4224ab64d60dd69e8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 Sep 2023 09:55:57 +0200 Subject: [PATCH 0067/1239] Enable CTP FLP processing by default on EPNS --- DATA/common/setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 961bc79e4..47257471a 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -122,7 +122,7 @@ else # Defaults when running on the EPN if [[ -z "${SHMTHROW:-}" ]]; then export SHMTHROW=0; fi if [[ -z "${TIMEFRAME_SHM_LIMIT:-}" ]]; then export TIMEFRAME_SHM_LIMIT=$(( $SHMSIZE / 2 )); fi if [[ -z "${EDJSONS_DIR:-}" ]]; then export EDJSONS_DIR="/scratch/services/ed/jsons_${RUNTYPE}"; fi - if [[ -z "${WORKFLOW_DETECTORS_FLP_PROCESSING+x}" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING="TOF"; fi # Current default in sync processing is that FLP processing is only enabled for TOF + if [[ -z "${WORKFLOW_DETECTORS_FLP_PROCESSING+x}" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING="TOF,CTP"; fi # Current default in sync processing is that FLP processing is only enabled for TOF if [[ -z "${GEN_TOPO_AUTOSCALE_PROCESSES:-}" ]]; then export GEN_TOPO_AUTOSCALE_PROCESSES=1; fi # On the EPN we should make sure to always use the node to the full extent fi # Some more options for running on the EPN From b81cd913546f16512d0710c143f9f29d5342e626 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 Sep 2023 13:02:10 +0200 Subject: [PATCH 0068/1239] GEN_TOPO_MI100_NODES should be defaulted to -1 --- DATA/tools/epn/gen_topo.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 5d2fa25a9..a8576f20a 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -36,7 +36,7 @@ if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='/usr/local/bin/epn-topo-alloc' fi -# Extra arguments for topology generation +# Extra arguments for topology merger if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" ]]; then if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS="--recozone staging-mi50 --reco100zone staging-mi100 --calibzone calib" @@ -44,6 +44,7 @@ if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" ]]; then export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS="--recozone online-mi50 --reco100zone online-mi100 --calibzone calib" fi fi +if [[ -z "$GEN_TOPO_MI100_NODES" ]]; then export GEN_TOPO_MI100_NODES=-1; fi # GEN_TOPO_RUN_HOME is a debug setting used in some tests. This is not needed for online running. if [[ "0$GEN_TOPO_RUN_HOME" == "01" ]]; then From ab3193be0a630f0ce6ffc00061f8264779f7e357 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 5 Sep 2023 13:06:07 +0200 Subject: [PATCH 0069/1239] setenv.sh: fix default for IS_TRIGGERED_DATA... --- DATA/common/setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 47257471a..e2ea083cc 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -93,7 +93,7 @@ if [[ -z "${EPNSYNCMODE:-}" ]]; then export EPNSYNCMODE=0; fi # if [[ -z "${BEAMTYPE:-}" ]]; then export BEAMTYPE=PbPb; fi # Beam type, must be PbPb, pp, pPb, cosmic, technical if [[ -z "${RUNTYPE:-}" ]]; then export RUNTYPE=Standalone; fi # Run Type, standalone for local tests, otherwise PHYSICS, COSMICS, TECHNICAL, SYNTHETIC if [[ -z "${IS_SIMULATED_DATA:-}" ]]; then export IS_SIMULATED_DATA=1; fi # processing simulated data -if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=1; fi # processing triggered data (TPC triggered instead of continuous) +if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi # processing triggered data (TPC triggered instead of continuous) if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing From 9d20b3b30e681f8cfaaefd14edbf1d8cc8384e3d Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Tue, 5 Sep 2023 14:57:36 +0200 Subject: [PATCH 0070/1239] Add HMP reco tasks in O2DPG MC workflow --- DATA/common/getCommonArgs.sh | 0 MC/bin/o2dpg_sim_workflow.py | 12 ++++++++++++ 2 files changed, 12 insertions(+) mode change 100644 => 100755 DATA/common/getCommonArgs.sh diff --git a/DATA/common/getCommonArgs.sh b/DATA/common/getCommonArgs.sh old mode 100644 new mode 100755 diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 25a939551..482b08f74 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1088,6 +1088,15 @@ def getDigiTaskName(det): MFTMCHMATCHTraintask['cmd']+= getDPL_global_options() workflow['stages'].append(MFTMCHMATCHTraintask) + # HMP tasks + HMPRECOtask = createTask(name='hmpreco_'+str(tf), needs=[getDigiTaskName('HMP')], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') + HMPRECOtask['cmd'] = '${O2_ROOT}/bin/o2-hmpid-digits-to-clusters-workflow ' + getDPL_global_options(ccdbbackend=False) + putConfigValuesNew() + workflow['stages'].append(HMPRECOtask) + + HMPMATCHtask = createTask(name='hmpmatch_'+str(tf), needs=[HMPRECOtask['name'],ITSTPCMATCHtask['name'],TOFTPCMATCHERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') + HMPMATCHtask['cmd'] = '${O2_ROOT}/bin/o2-hmpid-matcher-workflow ' + getDPL_global_options() + putConfigValuesNew() + workflow['stages'].append(HMPMATCHtask) + # Take None as default, we only add more if nothing from anchorConfig pvfinder_sources = anchorConfig.get("o2-primary-vertexing-workflow-options",{}).get("vertexing-sources", None) pvfinder_matching_sources = anchorConfig.get("o2-primary-vertexing-workflow-options",{}).get("vertex-track-matching-sources", None) @@ -1317,6 +1326,9 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if isActive('MID') and isActive('MCH'): aodneeds += [ MCHMIDMATCHtask['name'] ] aodinfosources += ',MCH-MID' + if isActive('HMP'): + aodneeds += [ HMPMATCHtask['name'] ] + aodinfosources += ',HMP' if args.with_ZDC and isActive('ZDC'): aodneeds += [ ZDCRECOtask['name'] ] aodinfosources += ',ZDC' From afd2d18fec5ce162b38ea42fdfd74597f6a7d6d6 Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 4 Sep 2023 17:59:58 +0200 Subject: [PATCH 0071/1239] Fix pre-collision context generation when anchoring --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 482b08f74..e9d15a738 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -593,7 +593,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): PbPbXSec=8. # expected PbPb cross section QEDXSecExpected=35237.5 # expected magnitude of QED cross section PreCollContextTask=createTask(name='precollcontext_' + str(tf), needs=precollneeds, tf=tf, cwd=timeframeworkdir, cpu='1') - PreCollContextTask['cmd']='${O2_ROOT}/bin/o2-steer-colcontexttool -i ' + signalprefix + ',' + str(INTRATE) + ',' + str(args.ns) + ':' + str(args.ns) + ' --show-context ' + ' --timeframeID ' + str(tf-1 + int(args.production_offset)*NTIMEFRAMES) + ' --orbitsPerTF ' + str(orbitsPerTF) + ' --orbits ' + str(orbitsPerTF) + ' --seed ' + str(TFSEED) + ' --noEmptyTF' + PreCollContextTask['cmd']='${O2_ROOT}/bin/o2-steer-colcontexttool -i ' + signalprefix + ',' + str(INTRATE) + ',' + str(args.ns) + ':' + str(args.ns) + ' --show-context ' + ' --timeframeID ' + str(tf-1 + int(args.production_offset)*NTIMEFRAMES) + ' --orbitsPerTF ' + str(orbitsPerTF) + ' --orbits ' + str(orbitsPerTF) + ' --seed ' + str(TFSEED) + ' --noEmptyTF --first-orbit ' + str(args.first_orbit) PreCollContextTask['cmd'] += ' --bcPatternFile ccdb' # <--- the object should have been set in (local) CCDB if includeQED: qedrate = INTRATE * QEDXSecExpected / PbPbXSec # hadronic interaction rate * cross_section_ratio From 4b401b0f74cc711511c6a0463505de74102e1608 Mon Sep 17 00:00:00 2001 From: Luca Barioglio Date: Tue, 5 Sep 2023 22:45:05 +0200 Subject: [PATCH 0072/1239] Randomise PDG code (#1166) --- .../ini/tests/GeneratorLFHyperNucleiPbPbGap.C | 35 ++++++++----------- .../ini/tests/GeneratorLFHyperNucleippGap.C | 35 ++++++++----------- ...generator_pythia8_longlived_gaptriggered.C | 18 +++++----- 3 files changed, 37 insertions(+), 51 deletions(-) diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleiPbPbGap.C b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleiPbPbGap.C index 00ca43a3c..0f51674fb 100644 --- a/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleiPbPbGap.C +++ b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleiPbPbGap.C @@ -1,14 +1,14 @@ int External() { std::string path{"o2sim_Kine.root"}; - int numberOfInjectedSignalsPerEvent{10}; - std::vector injectedPDGs = {1000010020, -1000010020, + std::vector possiblePDGs = {1000010020, -1000010020, 1000010030, -1000010030, 1000020030, -1000020030, 1000020040, -1000020040, - 1010010030, -1010010030}; + 1010010030, -1010010030, + 1010010040, -1010010040}; - auto nInjection = injectedPDGs.size(); + int nPossiblePDGs = possiblePDGs.size(); TFile file(path.c_str(), "READ"); if (file.IsZombie()) @@ -26,11 +26,7 @@ int External() std::vector *tracks{}; tree->SetBranchAddress("MCTrack", &tracks); - std::vector nSignal; - for (int i = 0; i < nInjection; i++) - { - nSignal.push_back(0); - } + std::vector injectedPDGs; auto nEvents = tree->GetEntries(); for (int i = 0; i < nEvents; i++) @@ -40,26 +36,23 @@ int External() { auto track = tracks->at(idxMCTrack); auto pdg = track.GetPdgCode(); - auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); - int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG - if (it != injectedPDGs.end()) // found + auto it = std::find(possiblePDGs.begin(), possiblePDGs.end(), pdg); + if (it != possiblePDGs.end() && track.isPrimary()) // found { - // count signal PDG - nSignal[index]++; + injectedPDGs.push_back(pdg); } } } std::cout << "--------------------------------\n"; std::cout << "# Events: " << nEvents << "\n"; - for (int i = 0; i < nInjection; i++) + if(injectedPDGs.empty()){ + std::cerr << "No injected particles\n"; + return 1; // At least one of the injected particles should be generated + } + for (int i = 0; i < nPossiblePDGs; i++) { std::cout << "# Injected nuclei \n"; - std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; - if (nSignal[i] == 0) - { - std::cerr << "No generated: " << injectedPDGs[i] << "\n"; - return 1; // At least one of the injected particles should be generated - } + std::cout << possiblePDGs[i] << ": " << std::count(injectedPDGs.begin(), injectedPDGs.end(), possiblePDGs[i]) << "\n"; } return 0; } diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGap.C b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGap.C index 54810d6cf..0f51674fb 100644 --- a/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGap.C +++ b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGap.C @@ -1,14 +1,14 @@ int External() { std::string path{"o2sim_Kine.root"}; - int numberOfInjectedSignalsPerEvent{1}; - std::vector injectedPDGs = {1000010020, -1000010020, + std::vector possiblePDGs = {1000010020, -1000010020, 1000010030, -1000010030, 1000020030, -1000020030, 1000020040, -1000020040, - 1010010030, -1010010030}; + 1010010030, -1010010030, + 1010010040, -1010010040}; - auto nInjection = injectedPDGs.size(); + int nPossiblePDGs = possiblePDGs.size(); TFile file(path.c_str(), "READ"); if (file.IsZombie()) @@ -26,11 +26,7 @@ int External() std::vector *tracks{}; tree->SetBranchAddress("MCTrack", &tracks); - std::vector nSignal; - for (int i = 0; i < nInjection; i++) - { - nSignal.push_back(0); - } + std::vector injectedPDGs; auto nEvents = tree->GetEntries(); for (int i = 0; i < nEvents; i++) @@ -40,26 +36,23 @@ int External() { auto track = tracks->at(idxMCTrack); auto pdg = track.GetPdgCode(); - auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); - int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG - if (it != injectedPDGs.end()) // found + auto it = std::find(possiblePDGs.begin(), possiblePDGs.end(), pdg); + if (it != possiblePDGs.end() && track.isPrimary()) // found { - // count signal PDG - nSignal[index]++; + injectedPDGs.push_back(pdg); } } } std::cout << "--------------------------------\n"; std::cout << "# Events: " << nEvents << "\n"; - for (int i = 0; i < nInjection; i++) + if(injectedPDGs.empty()){ + std::cerr << "No injected particles\n"; + return 1; // At least one of the injected particles should be generated + } + for (int i = 0; i < nPossiblePDGs; i++) { std::cout << "# Injected nuclei \n"; - std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; - if (nSignal[i] == 0) - { - std::cerr << "No generated: " << injectedPDGs[i] << "\n"; - return 1; // At least one of the injected particles should be generated - } + std::cout << possiblePDGs[i] << ": " << std::count(injectedPDGs.begin(), injectedPDGs.end(), possiblePDGs[i]) << "\n"; } return 0; } diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C index ee96fe3aa..c34411fb7 100644 --- a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +++ b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C @@ -29,7 +29,6 @@ public: setPt(pt_min, pt_max); mMass = getMass(input_pdg); mGeneratedEvents = 0; - mInjectionIndex = 0; mAlternatingPDGsign = true; } @@ -60,7 +59,6 @@ public: mInverseTriggerRatio = input_trigger_ratio; mMass = getMass(mPdg); mGeneratedEvents = 0; - mInjectionIndex = 0; mAlternatingPDGsign = true; } @@ -126,11 +124,12 @@ public: if (mGeneratedEvents % mInverseTriggerRatio == 0) { static int sign = 1; - int currentPdg = mPdg[mInjectionIndex]; - double currentMass = mMass[mInjectionIndex]; - for (int i = 0; i < mNinjected[mInjectionIndex]; ++i) + int injectionIndex = (int)gRandom->Uniform(0, mPdg.size()); + int currentPdg = mPdg[injectionIndex]; + double currentMass = mMass[injectionIndex]; + for (int i = 0; i < mNinjected[injectionIndex]; ++i) { - const double pt = gRandom->Uniform(mPtMin[mInjectionIndex], mPtMax[mInjectionIndex]); + const double pt = gRandom->Uniform(mPtMin[injectionIndex], mPtMax[injectionIndex]); const double rapidity = gRandom->Uniform(mYmin, mYmax); const double phi = gRandom->Uniform(0, TMath::TwoPi()); const double px{pt * std::cos(phi)}; @@ -138,13 +137,15 @@ public: const double mt{std::hypot(pt, currentMass)}; const double pz{mt * std::sinh(rapidity)}; const double et{mt * std::cosh(rapidity)}; - sign *= 1 - 2 * mAlternatingPDGsign; + if (mAlternatingPDGsign) + { + sign *= 1 - 2 * (gRandom->Uniform() > 0.5); + } mParticles.push_back(TParticle(sign * currentPdg, 1, -1, -1, -1, -1, px, py, pz, et, 0., 0., 0., 0.)); // make sure status code is encoded properly. Transport flag will be set by default and we have nothing // to do since all pushed particles should be tracked. o2::mcutils::MCGenHelper::encodeParticleStatusAndTracking(mParticles.back()); } - mInjectionIndex = ++mInjectionIndex % (int)mPdg.size(); } mGeneratedEvents++; return true; @@ -165,7 +166,6 @@ private: // Control gap-triggering unsigned long long mGeneratedEvents; /// number of events generated so far - unsigned long long mInjectionIndex; /// index of the particle in mPDG to be injected int mInverseTriggerRatio; /// injection gap }; From e82fc010fdb9bcfa2b7cc1b5581f0001f166b0f4 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Wed, 6 Sep 2023 21:45:13 +0200 Subject: [PATCH 0073/1239] Enable TRD QC tracking task by default (#1219) --- DATA/production/qc-async/trd.json | 2 +- DATA/production/qc-sync/trd.json | 2 +- DATA/production/qc-workflow.sh | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index 65db89ddb..bf81c26b8 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -60,7 +60,7 @@ } }, "Tracking": { - "active": "false", + "active": "true", "className": "o2::quality_control_modules::trd::TrackingTask", "moduleName": "QcTRD", "detectorName": "TRD", diff --git a/DATA/production/qc-sync/trd.json b/DATA/production/qc-sync/trd.json index 9e14e542b..cdeb487f7 100644 --- a/DATA/production/qc-sync/trd.json +++ b/DATA/production/qc-sync/trd.json @@ -129,7 +129,7 @@ ] }, "Tracking": { - "active": "false", + "active": "true", "className": "o2::quality_control_modules::trd::TrackingTask", "moduleName": "QcTRD", "detectorName": "TRD", diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index a3853b988..b46e3f133 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -244,6 +244,10 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.active=false' add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.PHTrackMatch.active=false' fi + if has_matching_qc TPCTRD && has_detectors_reco TPC TRD; then # should be only enabled in async + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.dataSource.query=\"trackITSTPCTRD:TRD/MATCH_ITSTPC\;trigITSTPCTRD:TRD/TRGREC_ITSTPC\;trackTPCTRD:TRD/MATCH_TPC\;trigTPCTRD:TRD/TRGREC_TPC\"' + add_pipe_separated QC_DETECTOR_CONFIG_OVERRIDE '.qc.tasks.Tracking.taskParameters.trackSources=\"ITS-TPC-TRD,TPC-TRD\"' + fi fi fi From 7cefbe7a6c038de4cd784575a170063b4c32cb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Thu, 7 Sep 2023 05:13:43 -0500 Subject: [PATCH 0074/1239] Update analysis QC for the K0, format json (#1220) * Add pp900 inel config * Update analysis QC, format json --- .../o2dpg_analysis_test_workflow.py | 5 + .../generator/pythia8_inel_pp900gev.cfg | 18 ++ .../json/analysis-testing-data.json | 284 ++++++++++-------- 3 files changed, 175 insertions(+), 132 deletions(-) create mode 100644 MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 9ae7e0549..6a58f7192 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -214,6 +214,7 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis from ROOT import TFile froot = TFile.Open(input_aod, "READ") found_O2collision_001 = False + found_O2zdc_001 = False for i in froot.GetListOfKeys(): if "DF_" not in i.GetName(): continue @@ -223,8 +224,12 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis # print(j) if "O2collision_001" in j.GetName(): found_O2collision_001 = True + if "O2zdc_001" in j.GetName(): + found_O2zdc_001 = True if not found_O2collision_001: additional_workflows.append("o2-analysis-collision-converter --doNotSwap") + if not found_O2zdc_001: + additional_workflows.append("o2-analysis-zdc-converter") break if input_aod.endswith(".txt") and not input_aod.startswith("@"): input_aod = f"@{input_aod}" diff --git a/MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg b/MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg new file mode 100644 index 000000000..fdff89896 --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg @@ -0,0 +1,18 @@ +### beams +Beams:idA = 2212 # proton +Beams:idB = 2212 # proton +Beams:eCM = 900. # GeV + +### processes +SoftQCD:inelastic = on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### phase space cuts +PhaseSpace:pTHatMin = 0.000000 +PhaseSpace:pTHatMax = -1.000000 + +Random:setSeed = on +Random:seed = 0 \ No newline at end of file diff --git a/MC/config/analysis_testing/json/analysis-testing-data.json b/MC/config/analysis_testing/json/analysis-testing-data.json index 668854d2f..e910814a8 100644 --- a/MC/config/analysis_testing/json/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/analysis-testing-data.json @@ -1,4 +1,50 @@ { + "EMCClusterMonitorTask": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "0", + "processCollisions": "1", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "1", + "processCollisions": "0", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, "ambiguous-track-propagation": { "ccdb-url": "http://alice-ccdb.cern.ch", "geoPath": "GLO/Config/GeometryAligned", @@ -11,6 +57,71 @@ "processRun2": "false", "processRun3": "true" }, + "cell-monitor": { + "maxCellTimeMain": "100", + "minCellAmplitude": "0", + "minCellAmplitudeTimeHists": "0.3", + "minCellTimeMain": "-50", + "selectBCID": "all", + "vetoBCID": "" + }, + "emc-tmmonitor": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "hasPropagatedTracks": "1", + "maxTime": "20", + "minM02": "0.1", + "minTime": "-25", + "processCollisions": "1", + "selectBCID": "all", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + }, + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "usePionRejection": "0", + "vertexCut": "-1", + "vetoBCID": "" + }, + "emcal-correction-task": { + "clusterDefinition": "kV3Default", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellFraction": "0.97", + "exoticCellInCrossMinAmplitude": "0.1", + "exoticCellMinAmplitude": "4", + "hasPropagatedTracks": "0", + "hasShaperCorrection": "1", + "isMC": "0", + "logWeight": "4.5", + "maxMatchingDistance": "0.4", + "nonlinearityFunction": "DATA_TestbeamFinal_NoScale", + "processFull": "1", + "processMCFull": "0", + "processStandalone": "0", + "selectedCellType": "1", + "useWeightExotic": "0" + }, "event-selection-qa-task": { "isMC": "false", "processRun2": "false", @@ -60,35 +171,37 @@ "perf-k0s-resolution": { "etaBins": { "values": [ - 10, - -1, - 1 + "20", + "-1", + "1" ] }, - "eventSelection": "1", + "eventSelection": "true", "mBins": { "values": [ - 300, - 0.35, - 0.65 + "300", + "0.35", + "0.65" ] }, "nSigTPC": "10", "pTBins": { "values": [ - 200, - 0, - 10 + "200", + "0", + "10" ] }, "phiBins": { "values": [ - 18, - 0, - 6.2831853 + "54", + "0", + "6.2831853" ] }, "rapidity": "0.5", + "requireTRDneg": "0", + "requireTRDpos": "0", "v0lifetime": "3", "v0setting_cospa": "0.995", "v0setting_dcanegtopv": "0.1", @@ -96,6 +209,10 @@ "v0setting_dcav0dau": "1", "v0setting_radius": "0.9" }, + "pid-multiplicity": { + "processIU": "1", + "processStandard": "0" + }, "qa-efficiency": { "applyEvSel": "2", "do-al": "false", @@ -222,13 +339,27 @@ "targetNumberOfEvents": "10000000", "trackSelection": "1" }, + "qa-k0s-tracking-efficiency": { + "eventSelection": "true", + "nSigTPC": "10", + "nSigmaBins": { + "values": [ + "1000", + "-100", + "100" + ] + }, + "processIU": "true", + "rapidity": "0.5", + "v0cospa": "0.995" + }, "tof-signal": "", "tpc-pid-full": { - "autofetchNetworks": "1", + "autofetchNetworks": "true", "ccdb-timestamp": "0", "ccdb-url": "http://alice-ccdb.cern.ch", "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "1", + "enableNetworkOptimizations": "true", "networkPathCCDB": "Analysis/PID/TPC/ML", "networkPathLocally": "network.onnx", "networkSetNumThreads": "0", @@ -242,11 +373,9 @@ "pid-pi": "-1", "pid-pr": "-1", "pid-tr": "-1", - "useNetworkCorrection": "0" - }, - "pid-multiplicity": { - "processIU": "0", - "processStandard": "1" + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" }, "track-extension": { "processRun2": "false", @@ -263,123 +392,14 @@ }, "track-selection": { "compatibilityIU": "false", + "dcaSetup": "0", "etaMax": "0.8", "etaMin": "-0.8", "isRun3": "true", "itsMatching": "1", "produceFBextendedTable": "-1", + "produceTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" - }, - "emcal-correction-task": { - "nonlinearityFunction": "DATA_TestbeamFinal_NoScale", - "logWeight": "4.5", - "exoticCellMinAmplitude": "4", - "clusterDefinition": "kV3Default", - "useWeightExotic": "0", - "disableNonLin": "0", - "exoticCellDiffTime": "1e+06", - "exoticCellInCrossMinAmplitude": "0.1", - "maxMatchingDistance": "0.4", - "exoticCellFraction": "0.97", - "hasShaperCorrection": "1", - "isMC": "0", - "hasPropagatedTracks": "0", - "processFull": "1", - "selectedCellType": "1", - "processStandalone": "0", - "processMCFull": "0" - }, - "cell-monitor": { - "minCellTimeMain": "-50", - "maxCellTimeMain": "100", - "minCellAmplitudeTimeHists": "0.3", - "vetoBCID": "", - "selectBCID": "all", - "minCellAmplitude": "0" - }, - "EMCClusterMonitorTaskAmbiguous": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "0", - "doEventSel": "0", - "processAmbiguous": "1", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "EMCClusterMonitorTask": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "1", - "doEventSel": "0", - "processAmbiguous": "0", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "emc-tmmonitor": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "doEventSel": "0", - "maxTime": "20", - "clusterDefinition": "10", - "minM02": "0.1", - "vetoBCID": "", - "processCollisions": "1", - "hasPropagatedTracks": "1", - "usePionRejection": "0", - "minTime": "-25", - "tpcNsigmaPion": { - "values": [ - -3, - 3 - ] - }, - "selectBCID": "all", - "tpcNsigmaElectron": { - "values": [ - -1, - 3 - ] - }, - "vertexCut": "-1", - "tpcNsigmaBack": { - "values": [ - -10, - -4 - ] - } } -} +} \ No newline at end of file From 4750f15e3eaca59ac3325d0580fdf8e2c7cb8a99 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 8 Sep 2023 13:44:59 +0200 Subject: [PATCH 0075/1239] Set PYTHONPATH in the cmdline in gen_topo.sh --- DATA/tools/epn/gen_topo.sh | 4 ++-- DATA/tools/epn/gen_topo_o2dpg.sh | 3 +-- DATA/tools/parse | 4 +--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index a8576f20a..973dd64b5 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -28,12 +28,12 @@ if [[ "0$GEN_TOPO_ONTHEFLY" == "01" ]]; then export SHM_MANAGER_SHMID=1 ;fi # Command for topology merging if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_CMD" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_CMD='/usr/local/bin/epn-topo-merger' + export GEN_TOPO_ODC_EPN_TOPO_CMD='env - PYTHONPATH+=/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9/site-packages /usr/local/bin/epn-topo-merger' fi # Command for postprocessing of topology generation after topology caching if [[ -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then - export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='/usr/local/bin/epn-topo-alloc' + export GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD='env - PYTHONPATH+=/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9/site-packages /usr/local/bin/epn-topo-alloc' fi # Extra arguments for topology merger diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 914470048..e9aa5a03c 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -115,8 +115,7 @@ if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then fi TMP_POST_CACHING_CMD+=" -o $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml" echo "Running post-caching topo-merger command: $TMP_POST_CACHING_CMD" 1>&2 - # Run wih a custom PYTHONPATH in order to import the python packages needed for the topology merger - eval PYTHONPATH="${PYTHONPATH}:/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9" $TMP_POST_CACHING_CMD 1>&2 || { echo Error during EPN topology-merger resource allocation 1>&2; exit 1; } + eval $TMP_POST_CACHING_CMD 1>&2 || { echo Error during EPN topology-merger resource allocation 1>&2; exit 1; } mv -f $GEN_TOPO_WORKDIR/output.xml.new $GEN_TOPO_WORKDIR/output.xml 1>&2 fi diff --git a/DATA/tools/parse b/DATA/tools/parse index 1f8d82156..dd2169b52 100755 --- a/DATA/tools/parse +++ b/DATA/tools/parse @@ -252,11 +252,9 @@ for line in f: odccommand += ' -o ' + sys.argv[3] print('Running topology merger command', odccommand) # Add a custom PYTHONPATH in order to import the python packages needed for the topology merger - epn_topo_env = os.environ.copy() - epn_topo_env["PYTHONPATH"] = f"/usr/local/lib/python3.9/site-packages:/usr/local/lib64/python3.9/site-packages:{epn_topo_env['PYTHONPATH']}" starttime = time.time() try: - out = subprocess.check_output(odccommand, stderr=subprocess.STDOUT, shell=True, env=epn_topo_env) + out = subprocess.check_output(odccommand, stderr=subprocess.STDOUT, shell=True) print(out.decode('utf-8')) print('Topology merger finished successfully') except subprocess.CalledProcessError as e: From c24c021ddbf328951212c8b5f2225319c146e6c7 Mon Sep 17 00:00:00 2001 From: Jeremy Wilkinson Date: Tue, 12 Sep 2023 15:54:39 +0200 Subject: [PATCH 0076/1239] add bc converter for workflow tests --- MC/analysis_testing/o2dpg_analysis_test_workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 6a58f7192..6306787ac 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -215,6 +215,7 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis froot = TFile.Open(input_aod, "READ") found_O2collision_001 = False found_O2zdc_001 = False + found_O2bc_001 = False for i in froot.GetListOfKeys(): if "DF_" not in i.GetName(): continue @@ -226,10 +227,14 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis found_O2collision_001 = True if "O2zdc_001" in j.GetName(): found_O2zdc_001 = True + if "O2bc_001" in j.GetName(): + found_O2bc_001 = True if not found_O2collision_001: additional_workflows.append("o2-analysis-collision-converter --doNotSwap") if not found_O2zdc_001: additional_workflows.append("o2-analysis-zdc-converter") + if not found_O2bc_001: + additional_workflows.append("o2-analysis-bc-converter") break if input_aod.endswith(".txt") and not input_aod.startswith("@"): input_aod = f"@{input_aod}" From b0f932b1c5baea24203f13898637c08c6e60f190 Mon Sep 17 00:00:00 2001 From: wiechula <11199190+wiechula@users.noreply.github.com> Date: Wed, 13 Sep 2023 13:46:00 +0200 Subject: [PATCH 0077/1239] Make autoset_convertes more flexible (#1226) * allow checking files on alien * allow using also input AOD lists --- .../o2dpg_analysis_test_workflow.py | 73 ++++++++++++------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 6306787ac..45b32bacd 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -186,6 +186,47 @@ def add_analysis_post_processing_tasks(workflow): task["cmd"] = f"root -l -b -q {post_processing_macro}{cmd}" workflow.append(task) +def get_additional_workflows(input_aod): + additional_workflows = [] + + # Treat case we have a text file as input. Use the first line in this case + if input_aod.endswith(".txt"): + if input_aod.startswith("@"): + input_aod = input_aod[1:] + with open(input_aod) as f: + input_aod = f.readline().strip('\n') + + if input_aod.endswith(".root"): + from ROOT import TFile + if input_aod.startswith("alien://"): + from ROOT import TGrid + TGrid.Connect("alien") + froot = TFile.Open(input_aod, "READ") + found_O2collision_001 = False + found_O2zdc_001 = False + found_O2bc_001 = False + for i in froot.GetListOfKeys(): + if "DF_" not in i.GetName(): + continue + df_dir = froot.Get(i.GetName()) + # print(i) + for j in df_dir.GetListOfKeys(): + # print(j) + if "O2collision_001" in j.GetName(): + found_O2collision_001 = True + if "O2zdc_001" in j.GetName(): + found_O2zdc_001 = True + if "O2bc_001" in j.GetName(): + found_O2bc_001 = True + if not found_O2collision_001: + additional_workflows.append("o2-analysis-collision-converter --doNotSwap") + if not found_O2zdc_001: + additional_workflows.append("o2-analysis-zdc-converter") + if not found_O2bc_001: + additional_workflows.append("o2-analysis-bc-converter") + break + return additional_workflows + def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis", *, analyses_only=None, is_mc=True, needs=None, autoset_converters=False, include_disabled_analyses=False, timeout=None): """Add default analyses to user workflow @@ -206,39 +247,15 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis if specified, list of other tasks which need to be run before """ - additional_workflows = [] if not input_aod.startswith("alien://"): input_aod = abspath(input_aod) - if autoset_converters: # This is needed to run with the latest TAG of the O2Physics with the older data - if input_aod.endswith(".root"): - from ROOT import TFile - froot = TFile.Open(input_aod, "READ") - found_O2collision_001 = False - found_O2zdc_001 = False - found_O2bc_001 = False - for i in froot.GetListOfKeys(): - if "DF_" not in i.GetName(): - continue - df_dir = froot.Get(i.GetName()) - # print(i) - for j in df_dir.GetListOfKeys(): - # print(j) - if "O2collision_001" in j.GetName(): - found_O2collision_001 = True - if "O2zdc_001" in j.GetName(): - found_O2zdc_001 = True - if "O2bc_001" in j.GetName(): - found_O2bc_001 = True - if not found_O2collision_001: - additional_workflows.append("o2-analysis-collision-converter --doNotSwap") - if not found_O2zdc_001: - additional_workflows.append("o2-analysis-zdc-converter") - if not found_O2bc_001: - additional_workflows.append("o2-analysis-bc-converter") - break if input_aod.endswith(".txt") and not input_aod.startswith("@"): input_aod = f"@{input_aod}" + additional_workflows = [] + if autoset_converters: # This is needed to run with the latest TAG of the O2Physics with the older data + additional_workflows = get_additional_workflows(input_aod) + data_or_mc = ANALYSIS_VALID_MC if is_mc else ANALYSIS_VALID_DATA for ana in load_analyses(analyses_only, include_disabled_analyses=include_disabled_analyses): From 2fa8c6b44fed23029f263b0f4e2d366aa595e3f3 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Fri, 8 Sep 2023 14:39:12 +0200 Subject: [PATCH 0078/1239] [EMCAL-833] Include more observables at cluster and cell level in EMCAL trending --- DATA/production/qc-postproc-async/emc.json | 501 ++++++++++++++++++++- 1 file changed, 483 insertions(+), 18 deletions(-) diff --git a/DATA/production/qc-postproc-async/emc.json b/DATA/production/qc-postproc-async/emc.json index 2c863b221..6d64cb564 100644 --- a/DATA/production/qc-postproc-async/emc.json +++ b/DATA/production/qc-postproc-async/emc.json @@ -33,7 +33,7 @@ } }, "postprocessing": { - "CellsTrending": { + "CellsTrend": { "active": "true", "className": "o2::quality_control::postprocessing::SliceTrendingTask", "moduleName": "QualityControl", @@ -107,6 +107,158 @@ ], "reductorName": "o2::quality_control_modules::common::TH1SliceReductor", "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Cells", + "names": [ + "ncellsPerEventSupermodule" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "200." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Cells", + "names": [ + "ncellsPerEventSupermoduleWThr" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "20." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Cells", + "names": [ + "cellAmplitudeSupermodule_PHYS" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "50." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Cells", + "names": [ + "cellTimeSupermodule_PHYS" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "-400.", + "800." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" } ], "plots": [ @@ -125,7 +277,7 @@ "varexp": "NEventsPerTFPHYS.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "0:.5", "graphAxisLabel": ":Run number" }, { @@ -134,7 +286,7 @@ "varexp": "ncellsPerEventTot.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -143,7 +295,7 @@ "varexp": "ncellsPerEventEMCALTot.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -152,7 +304,7 @@ "varexp": "ncellsPerEventDCALTot.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -161,7 +313,7 @@ "varexp": "ncellPerEventTot_Thres.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -170,7 +322,7 @@ "varexp": "ncellPerEventEMCALTot_Thres.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -179,7 +331,7 @@ "varexp": "ncellPerEventDCALTot_Thres.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -189,7 +341,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (ns):Run number" + "graphAxisLabel": " (ns):Run number" }, { "name": "MeanEMCALCellTime", @@ -198,7 +350,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (ns):Run number" + "graphAxisLabel": " (ns):Run number" }, { "name": "MeanDCALCellTime", @@ -207,7 +359,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (ns):Run number" + "graphAxisLabel": " (ns):Run number" }, { "name": "MeanCellEnergy", @@ -216,7 +368,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" }, { "name": "MeanEMCALCellEnergy", @@ -225,7 +377,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" }, { "name": "MeanDCALCellEnergy", @@ -234,7 +386,98 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" + }, + { + "name": "WidthCellEnergy", + "title": "Width cell energy (EMCAL+DCAL)", + "varexp": "cellAmplitude_PHYS.stddevX:run", + "selection": "", + "option": "*L", + "graphErrors": "0.:.5", + "graphAxisLabel": "#sigma(E_{cell}) (GeV):Run number" + }, + { + "name": "WidthEMCALCellEnergy", + "title": "Width cell energy (EMCAL)", + "varexp": "cellAmplitudeEMCAL_PHYS.stddevX:run", + "selection": "", + "option": "*L", + "graphErrors": "0.:.5", + "graphAxisLabel": "#sigma(E_{cell}) (GeV):Run number" + }, + { + "name": "WidthDCALCellEnergy", + "title": "Width cell energy (DCAL)", + "varexp": "cellAmplitudeDCAL_PHYS.stddevX:run", + "selection": "", + "option": "*L", + "graphErrors": "0.:.5", + "graphAxisLabel": "#sigma(E_{cell}) (GeV):Run number" + }, + { + "name": "MeanNumberOfCellsSupermodule", + "title": "Mean number of cells / event per supermodule", + "varexp": "ncellsPerEventSupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": "N_{cells} / event:run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanNumberOfCellsThresholdSupermodule", + "title": "Mean number of cells (E > 500 MeV)/ event per supermodule", + "varexp": "ncellsPerEventSupermoduleWThr.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": "N_{cells} / event:run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanCellEnergySupermodule", + "title": "Mean cell energy per supermodule", + "varexp": "cellAmplitudeSupermodule_PHYS.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (GeV):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanCellTimeSupermodule", + "title": "Mean cell time per supermodule", + "varexp": "cellTimeSupermodule_PHYS.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (ns):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" } ], "initTrigger": [ @@ -253,7 +496,7 @@ "moduleName": "QualityControl", "detectorName": "EMC", "resumeTrend": "false", - "producePlotsOnUpdate": "false", + "producePlotsOnUpdate": "true", "dataSources": [ { "type": "repository", @@ -294,6 +537,122 @@ ] ], "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Clusters", + "names": [ + "ClusterEnergySupermodule", + "LeadingClusterEnergySupermodule" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "50." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Clusters", + "names": [ + "ClusterTimeSupermodule", + "LeadingClusterTimeSupermodule" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "-300.", + "300." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" + }, + { + "type": "repository", + "path": "EMC/MO/Clusters", + "names": [ + "NClustersPerEventSupermodule" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "200." + ], + [ + "-0.5", + "0.5", + "1.5", + "2.5", + "3.5", + "4.5", + "5.5", + "6.5", + "7.5", + "8.5", + "9.5", + "10.5", + "11.5", + "12.5", + "13.5", + "14.5", + "15.5", + "16.5", + "17.5", + "18.5", + "19.5." + ] + ], + "moduleName": "QcCommon" } ], "plots": [ @@ -303,7 +662,7 @@ "varexp": "NclustPerEvt.meanX:run", "selection": "", "option": "*L", - "graphErrors": "stddevX:.5", + "graphErrors": "errMeanX:.5", "graphAxisLabel": ":Run number" }, { @@ -369,7 +728,12 @@ "graphErrors": "errMeanY:0.5", "graphYRange": "", "graphXRange": "", - "graphAxisLabel": ":run" + "graphAxisLabel": ":run", + "legendNColums" : "1", + "legendTextSize": "14", + "legendObservableX": "E_{cell}", + "legendObservableY": "None", + "legendUnitX": "GeV" }, { "name": "NCellsClusterDCAL", @@ -380,7 +744,108 @@ "graphErrors": "errMeanY:0.5", "graphYRange": "", "graphXRange": "", - "graphAxisLabel": ":run" + "graphAxisLabel": ":run", + "legendNColums" : "1", + "legendTextSize": "14", + "legendObservableX": "E_{cell}", + "legendObservableY": "None", + "legendUnitX": "GeV" + }, + { + "name": "MeanClusterEnergySupermodule", + "title": "Mean cluster energy per supermodule", + "varexp": "ClusterEnergySupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (GeV):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanClusterTimeSupermodule", + "title": "Mean cluster time per supermodule", + "varexp": "ClusterTimeSupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (ns):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanClusterTimeSupermodule", + "title": "Mean cluster time per supermodule", + "varexp": "ClusterTimeSupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (ns):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanNumberOfClustersSupermodule", + "title": "Mean number of clusters / event per supermodule", + "varexp": "NClustersPerEventSupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": "N_{clusters} / event:run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanLeadingClusterEnergySupermodule", + "title": "Mean leading cluster energy per supermodule", + "varexp": "LeadingClusterEnergySupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (GeV):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" + }, + { + "name": "MeanLeadingClusterTimeSupermodule", + "title": "Mean leading cluster time per supermodule", + "varexp": "LeadingClusterTimeSupermodule.meanX:multigraphrun", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.5", + "graphYRange": "", + "graphXRange": "", + "graphAxisLabel": " (ns):run", + "legendNColums" : "2", + "legendTextSize": "14", + "legendObservableX": "None", + "legendObservableY": "Supermodule", + "legendCentmodeY": "True" } ], "initTrigger": [ From 30250f6737d942534fad05d061ce9f36b1f8e4f7 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Wed, 13 Sep 2023 21:53:43 +0200 Subject: [PATCH 0079/1239] Tpc laser (#1225) * Change of laser workflows for Dense packade format * Add laser raw-filter workflow and move pattern_generator workflow to TPC/test * laser raw-filter cleanup * Bugfix * Add new options for laser run * fixed for tpc-laser-multi * Remove only-publsihed-on-eos * cosmetic correction * add optin to add publish ony on eos --- DATA/production/calib/tpc-laser-aggregator.sh | 21 ++++++++++++------- DATA/production/calib/tpc-laser-filter.sh | 9 ++++++-- DATA/production/calib/tpc-laser.sh | 13 +++++++++--- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/DATA/production/calib/tpc-laser-aggregator.sh b/DATA/production/calib/tpc-laser-aggregator.sh index 5ad585363..2fc461a38 100755 --- a/DATA/production/calib/tpc-laser-aggregator.sh +++ b/DATA/production/calib/tpc-laser-aggregator.sh @@ -6,7 +6,7 @@ source common/getCommonArgs.sh PROXY_INSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSREFS" -CALIB_CONFIG= "TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" \ +CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" CCDB_PATH="http://o2-ccdb.internal" @@ -19,6 +19,7 @@ publish_after=440 min_tracks=0 num_lanes=36 + if [[ ! -z ${TPC_CALIB_MAX_EVENTS:-} ]]; then max_events=${TPC_CALIB_MAX_EVENTS} fi @@ -33,7 +34,14 @@ if [[ ! -z ${TPC_CALIB_LANES_PAD_RAW:-} ]]; then num_lanes=${TPC_CALIB_LANES_PAD_RAW} fi -EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes #{num_lanes} --check-calib-infos" +EXTRA_CONFIG="--calib-type ce --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes ${num_lanes} --check-calib-infos" + +EXTRA_CONFIG_TRACKS="" + +if [[ ${TPC_CALIB_TRACKS_PUBLISH_EOS:-} == 1 ]]; then + EXTRA_CONFIG_TRACKS="--only-publish-on-eos" +fi + o2-dpl-raw-proxy $ARGS_ALL \ @@ -41,12 +49,11 @@ o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "${PROXY_INSPEC}" \ --network-interface ib0 \ --channel-config "name=tpc-laser-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq" \ - | o2-tpc-calib-laser-tracks ${ARGS_ALL} \ - --use-filtered-tracks --only-publish-on-eos --min-tfs=${min_tracks}\ + | o2-tpc-calib-laser-tracks ${ARGS_ALL} --use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks} \ + --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \ | o2-tpc-calib-pad-raw ${ARGS_ALL} \ - --configKeyValues ${CALIB_CONFIG} \ - ${EXTRA_CONFIG} \ + --configKeyValues ${CALIB_CONFIG} ${EXTRA_CONFIG} \ | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ --ccdb-path ${CCDB_PATH} \ - | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} + diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 2c4344e54..e037197b9 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -46,6 +46,10 @@ PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSRE LASER_DECODER_ADD='' +HOST=localhost + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" fi @@ -66,13 +70,14 @@ o2-dpl-raw-proxy $ARGS_ALL \ --disable-mc \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ $GPU_CONFIG \ - --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField" \ - --configKeyValues "$ARGS_ALL_CONFIG;align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1" \ + --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \ + --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35" \ | o2-tpc-laser-track-filter $ARGS_ALL \ | o2-dpl-output-proxy ${ARGS_ALL} \ --dataspec "$PROXY_OUTSPEC" \ --proxy-name tpc-laser-input-proxy \ --proxy-channel-name tpc-laser-input-proxy \ --channel-config "name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0" \ + | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index 8b1b33050..76ac79ec0 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -77,6 +77,13 @@ if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root --decoder-type 0" fi +EXTRA_CONFIG_TRACKS="" + +if [[ ${TPC_CALIB_TRACKS_PUBLISH_EOS:-} == 1 ]]; then + EXTRA_CONFIG_TRACKS="--only-publish-on-eos" +fi + + o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ @@ -92,10 +99,10 @@ o2-dpl-raw-proxy $ARGS_ALL \ --disable-mc \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ $GPU_CONFIG \ - --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField" \ - --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1" \ + --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \ + --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35" \ | o2-tpc-laser-track-filter $ARGS_ALL \ - | o2-tpc-calib-laser-tracks $ARGS_ALL --use-filtered-tracks --only-publish-on-eos --min-tfs=${min_tracks}\ + | o2-tpc-calib-laser-tracks $ARGS_ALL --use-filtered-tracks ${EXTRA_CONFIG_TRACKS} --min-tfs=${min_tracks}\ | o2-tpc-calib-pad-raw ${ARGS_ALL} \ --configKeyValues ${CALIB_CONFIG} ${EXTRA_CONFIG} \ | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ From 404d2e50856809bc365c7ff6dafec8bfcffade11 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Thu, 14 Sep 2023 10:41:13 +0200 Subject: [PATCH 0080/1239] [EMCAL-833] Adding trending for mean cluster time --- DATA/production/qc-postproc-async/emc.json | 122 +++++++++++++++++++-- 1 file changed, 115 insertions(+), 7 deletions(-) diff --git a/DATA/production/qc-postproc-async/emc.json b/DATA/production/qc-postproc-async/emc.json index 6d64cb564..9908bb047 100644 --- a/DATA/production/qc-postproc-async/emc.json +++ b/DATA/production/qc-postproc-async/emc.json @@ -33,7 +33,7 @@ } }, "postprocessing": { - "CellsTrend": { + "CellTrend": { "active": "true", "className": "o2::quality_control::postprocessing::SliceTrendingTask", "moduleName": "QualityControl", @@ -490,7 +490,7 @@ "once" ] }, - "ClustersTrending": { + "ClustersTrend": { "active": "true", "className": "o2::quality_control::postprocessing::SliceTrendingTask", "moduleName": "QualityControl", @@ -505,6 +505,12 @@ "ClustE_All", "ClustE_DCal", "ClustE_EMCal", + "ClusterELeading_All", + "ClusterELeading_EMCal", + "ClusterELeading_DCal", + "ClusterTimeLeading_All", + "ClusterTimeLeading_EMCal", + "ClusterTimeLeading_DCal", "M02_All", "M20_DCal", "M20_EMCal", @@ -538,6 +544,27 @@ ], "moduleName": "QcCommon" }, + { + "type": "repository", + "path": "EMC/MO/Clusters", + "names": [ + "Time_All", + "Time_EMCal", + "Time_DCal" + ], + "reductorName": "o2::quality_control_modules::common::TH2SliceReductor", + "axisDivision": [ + [ + "0.", + "5.0" + ], + [ + "-300.", + "300." + ] + ], + "moduleName": "QcCommon" + }, { "type": "repository", "path": "EMC/MO/Clusters", @@ -672,7 +699,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" }, { "name": "MeanEMCALClusterEnergy", @@ -681,7 +708,7 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" }, { "name": "MeanDCALClusterEnergy", @@ -690,7 +717,88 @@ "selection": "", "option": "*L", "graphErrors": "errMeanX:.5", - "graphAxisLabel": " (GeV):Run number" + "graphAxisLabel": " (GeV):Run number" + }, + { + "name": "MeanLeadingClusterEnergy", + "title": "Mean leading cluster energy (EMCAL+DCAL)", + "varexp": "ClusterELeading_All.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (GeV):Run number" + }, + { + "name": "MeanLeadingEMCALClusterEnergy", + "title": "Mean leading cluster energy (EMCAL)", + "varexp": "ClusterELeading_EMCal.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (GeV):Run number" + }, + { + "name": "MeanLeadingDCALClusterEnergy", + "title": "Mean leading cluster energy (DCAL)", + "varexp": "ClusterELeading_DCal.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (GeV):Run number" + }, + { + "name": "MeanClusterTime", + "title": "Mean cluster time (EMCAL+DCAL)", + "varexp": "Time_All.meanY:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanY:.5", + "graphAxisLabel": " (ns):Run number" + }, + { + "name": "MeanEMCALClusterTime", + "title": "Mean cluster time (EMCAL)", + "varexp": "Time_EMCal.meanY:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanY:.5", + "graphAxisLabel": " (ns):Run number" + }, + { + "name": "MeanDCALClusterTime", + "title": "Mean cluster time (DCAL)", + "varexp": "Time_DCal.meanY:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanY:.5", + "graphAxisLabel": " (ns):Run number" + }, + { + "name": "MeanLeadingClusterTime", + "title": "Mean leading cluster time (EMCAL+DCAL)", + "varexp": "ClusterTimeLeading_All.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (ns):Run number" + }, + { + "name": "MeanLeadingEMCALClusterTime", + "title": "Mean leading cluster time (EMCAL)", + "varexp": "ClusterTimeLeading_EMCal.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (ns):Run number" + }, + { + "name": "MeanLeadingDCALClusterTime", + "title": "Mean leading cluster time (DCAL)", + "varexp": "ClusterTimeLeading_DCal.meanX:run", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:.5", + "graphAxisLabel": " (ns):Run number" }, { "name": "MeanClusterM02", @@ -731,7 +839,7 @@ "graphAxisLabel": ":run", "legendNColums" : "1", "legendTextSize": "14", - "legendObservableX": "E_{cell}", + "legendObservableX": "N_{cell}", "legendObservableY": "None", "legendUnitX": "GeV" }, @@ -747,7 +855,7 @@ "graphAxisLabel": ":run", "legendNColums" : "1", "legendTextSize": "14", - "legendObservableX": "E_{cell}", + "legendObservableX": "N_{cell}", "legendObservableY": "None", "legendUnitX": "GeV" }, From eb8eb28e628df4dc04f0e264fde20b74641c7bf0 Mon Sep 17 00:00:00 2001 From: Antonio FRANCO Date: Thu, 14 Sep 2023 16:18:38 +0200 Subject: [PATCH 0081/1239] Hmpid pedestals v3 (#1218) * Modernization of standalone detector workflows * Fix error in ARGS_ALL env. variable use * Fix the o2-dpl-raw-proxy line * Rebase to master --------- Co-authored-by: Antonio Franco Co-authored-by: Ole Schmidt --- DATA/production/calib/hmp-pedestals-processing.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DATA/production/calib/hmp-pedestals-processing.sh b/DATA/production/calib/hmp-pedestals-processing.sh index ef6fcdb10..6c05350a4 100755 --- a/DATA/production/calib/hmp-pedestals-processing.sh +++ b/DATA/production/calib/hmp-pedestals-processing.sh @@ -11,6 +11,7 @@ # HMP_NODCSCCDB_REC : True if we want disable DCS CCDB recording [=False] # HMP_FILES_REC : True if we want store on files (Only for debug) [=False] # +# 14/09/2023 - rebase # # Auth. A.Franco - INFN Sez.BARI - ITALY # ------------------------------------------------------------------------ @@ -73,8 +74,8 @@ SPEC_PARAM+="--pedestals-tag ${HMP_PED_TAG} --sigmacut ${HMP_SIGMACUT}" # Here we compose the workflow # Start with an empty workflow WORKFLOW= -add_W o2-dpl-raw-proxy "$ARGS_ALL --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\"" -add_W o2-hmpid-raw-to-pedestals-workflow "${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --fast-decode $SPEC_PARAM" +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\"" "" 0 +add_W o2-hmpid-raw-to-pedestals-workflow "--fast-decode $SPEC_PARAM" # Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" From 70766713f6faf50e06367f54b44320b31885e64a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 15 Sep 2023 22:31:14 +0200 Subject: [PATCH 0082/1239] Distinguish between ROOT_INPUT and DIGIT_ROOT_INPUT --- DATA/common/setenv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index e2ea083cc..4298b4c59 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -147,6 +147,7 @@ fi LIST_OF_ASYNC_RECO_STEPS="MID MCH MFT FDD FV0 ZDC HMP" DISABLE_DIGIT_ROOT_INPUT="--disable-root-input" +DISABLE_ROOT_INPUT="--disable-root-input" : ${DISABLE_DIGIT_CLUSTER_INPUT="--clusters-from-upstream"} # Special detector related settings From 0f0b27e4738894065969117e6a89ecfd2c5ec2be Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 18 Sep 2023 17:12:03 +0200 Subject: [PATCH 0083/1239] ptCutoff decreased to 0.3 for TRD (#1231) --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index f1899812b..8a9c2bdb3 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -404,7 +404,7 @@ export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_ma export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching -export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" +export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;GPU_rec_trd.minTrackPt=0.3;" # ad-hoc settings for FT0 export ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow --ft0-reconstructor" From 3eb0b5a20dac6cd7f9b00919d858dcc0da989d94 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 18 Sep 2023 21:25:41 +0200 Subject: [PATCH 0084/1239] GPU: Reduce GPUMEMSIZE to stay in lower 16 GB of MI50 GPU --- DATA/common/setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 4298b4c59..6a601a7af 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -70,7 +70,7 @@ if [[ -z "${TFDELAY:-}" ]]; then export TFDELAY=0; fi # if [[ -z "${GPUTYPE:-}" ]]; then export GPUTYPE=CPU; fi # GPU Tracking backend to use, can be CPU / CUDA / HIP / OCL / OCL2 if [[ -z "${DDSHMSIZE:-}" ]]; then export DDSHMSIZE=$(( 8 << 10 )); fi # Size of shared memory for DD Input if [[ -z "${DDHDRSIZE:-}" ]]; then export DDHDRSIZE=$(( 1 << 10 )); fi # Size of shared memory for DD Input -if [[ -z "${GPUMEMSIZE:-}" ]]; then export GPUMEMSIZE=$(( 24 << 30 )); fi # Size of allocated GPU memory (if GPUTYPE != CPU) +if [[ -z "${GPUMEMSIZE:-}" ]]; then export GPUMEMSIZE=$(( 14 << 30 )); fi # Size of allocated GPU memory (if GPUTYPE != CPU) if [[ -z "${HOSTMEMSIZE:-}" ]]; then export HOSTMEMSIZE=0; fi # Size of allocated host memory for GPU reconstruction (0 = default) if [[ -z "${CREATECTFDICT:-}" ]]; then export CREATECTFDICT=0; fi # Create CTF dictionary if [[ -z "${SAVECTF:-}" ]]; then export SAVECTF=0; fi # Save the CTF to a ROOT file From b44a1a5364ce09b279ebcbf32f0940e51401d46e Mon Sep 17 00:00:00 2001 From: Giovanni Malfattore <89481844+giovannimalfattore@users.noreply.github.com> Date: Tue, 19 Sep 2023 18:58:03 +0200 Subject: [PATCH 0085/1239] Add TPC-qa to PIDFull task (#1229) * Add TPC-qa to PIDFull task * Add TOF-qa settings to PIDFull task --- .../json/analyses_config.json | 1 + .../json/analysis-testing-data.json | 162 ++++++++++++++++++ .../json/analysis-testing-mc.json | 162 ++++++++++++++++++ 3 files changed, 325 insertions(+) diff --git a/MC/config/analysis_testing/json/analyses_config.json b/MC/config/analysis_testing/json/analyses_config.json index 4b646065b..a82a588ab 100644 --- a/MC/config/analysis_testing/json/analyses_config.json +++ b/MC/config/analysis_testing/json/analyses_config.json @@ -93,6 +93,7 @@ "o2-analysis-pid-tof-qa-evtime", "o2-analysis-pid-tof-beta", "o2-analysis-pid-tof-qa-beta", + "o2-analysis-pid-tpc-qa", "o2-analysis-pid-tpc-base", "o2-analysis-pid-tpc-full"] }, diff --git a/MC/config/analysis_testing/json/analysis-testing-data.json b/MC/config/analysis_testing/json/analysis-testing-data.json index e910814a8..6b1ad0ad5 100644 --- a/MC/config/analysis_testing/json/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/analysis-testing-data.json @@ -354,6 +354,168 @@ "v0cospa": "0.995" }, "tof-signal": "", + "tpc-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "dEdxBins": { + "values": [ + 3000, + 0, + 1000 + ] + }, + "applyRapidityCut": "0", + "processFullWithTOFDeuteron": "0", + "phiBins": { + "values": [ + 50, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "1", + "processKaon": "0", + "processAlpha": "0", + "processFullWithTOFPion": "0", + "processFullHelium3": "1", + "minP": "0.01", + "enableDeDxPlot": "1", + "processFullWithTOFElectron": "0", + "logAxis": "1", + "processFullWithTOFAlpha": "0", + "processFullKaon": "1", + "processMuon": "0", + "processFullDeuteron": "1", + "processFullWithTOFMuon": "0", + "processFullWithTOFProton": "0", + "maxP": "5", + "processFullAlpha": "0", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "processFullWithTOFKaon": "0", + "minTPCNcls": "0", + "processFullMuon": "1", + "processFullWithTOFTriton": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullWithTOFHelium3": "0", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 100, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tof-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "applyRapidityCut": "1", + "phiBins": { + "values": [ + 25, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "0", + "processKaon": "0", + "processAlpha": "0", + "processFullHelium3": "1", + "minP": "0.1", + "logAxis": "0", + "processFullKaon": "1", + "produceDeltaTEtaPhiMap": "0", + "processMuon": "0", + "processFullDeuteron": "1", + "maxP": "5", + "processFullAlpha": "0", + "ptDeltaTEtaPhiMap": "3", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", + "processFullMuon": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 200, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, "tpc-pid-full": { "autofetchNetworks": "true", "ccdb-timestamp": "0", diff --git a/MC/config/analysis_testing/json/analysis-testing-mc.json b/MC/config/analysis_testing/json/analysis-testing-mc.json index 08f0c7701..62209a027 100644 --- a/MC/config/analysis_testing/json/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/analysis-testing-mc.json @@ -1118,6 +1118,168 @@ "processOnlyBCs": "true" }, "tof-signal": "", + "tpc-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "dEdxBins": { + "values": [ + 3000, + 0, + 1000 + ] + }, + "applyRapidityCut": "0", + "processFullWithTOFDeuteron": "0", + "phiBins": { + "values": [ + 50, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "1", + "processKaon": "0", + "processAlpha": "0", + "processFullWithTOFPion": "0", + "processFullHelium3": "1", + "minP": "0.01", + "enableDeDxPlot": "1", + "processFullWithTOFElectron": "0", + "logAxis": "1", + "processFullWithTOFAlpha": "0", + "processFullKaon": "1", + "processMuon": "0", + "processFullDeuteron": "1", + "processFullWithTOFMuon": "0", + "processFullWithTOFProton": "0", + "maxP": "5", + "processFullAlpha": "0", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "processFullWithTOFKaon": "0", + "minTPCNcls": "0", + "processFullMuon": "1", + "processFullWithTOFTriton": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullWithTOFHelium3": "0", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 100, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tof-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "applyRapidityCut": "1", + "phiBins": { + "values": [ + 25, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "0", + "processKaon": "0", + "processAlpha": "0", + "processFullHelium3": "1", + "minP": "0.1", + "logAxis": "0", + "processFullKaon": "1", + "produceDeltaTEtaPhiMap": "0", + "processMuon": "0", + "processFullDeuteron": "1", + "maxP": "5", + "processFullAlpha": "0", + "ptDeltaTEtaPhiMap": "3", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", + "processFullMuon": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 200, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, "tpc-pid-full": { "autofetchNetworks": "1", "ccdb-timestamp": "0", From 73f9f424bf85d426cbcc4767f38b115298cafa11 Mon Sep 17 00:00:00 2001 From: shahoian Date: Fri, 22 Sep 2023 17:24:12 +0200 Subject: [PATCH 0086/1239] Complete TPC_CORR_SCALING explanation with new options --- DATA/common/setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 6a601a7af..2f2ca7491 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -97,7 +97,7 @@ if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing -if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --require-ctp-lumi, --corrmap-lumi-mean , --corrmap-lumi-inst +if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --require-ctp-lumi, --corrmap-lumi-mean , --corrmap-lumi-inst , --ctp-lumi-factor , --ctp-lumi-source if [[ $EPNSYNCMODE == 0 ]]; then if [[ -z "${SHMSIZE:-}" ]]; then export SHMSIZE=$(( 8 << 30 )); fi # Size of shared memory for messages if [[ -z "${NGPUS:-}" ]]; then export NGPUS=1; fi # Number of GPUs to use, data distributed round-robin From 6a685fba3c7645cd49393b178b87c32c45ea4611 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Fri, 22 Sep 2023 18:04:24 +0200 Subject: [PATCH 0087/1239] [EMCAL-756] Async QC configuration for PbPb Adaption of task parameters (mainly histogram ranges of multiplicity-dependent observables) based on data from run 543218 --- DATA/production/qc-async/emc_PbPb.json | 96 ++++++++++++++++++++++++++ DATA/production/qc-workflow.sh | 7 +- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 DATA/production/qc-async/emc_PbPb.json diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json new file mode 100644 index 000000000..a91957c10 --- /dev/null +++ b/DATA/production/qc-async/emc_PbPb.json @@ -0,0 +1,96 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "qcdb.cern.ch:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2" + }, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "alio2-cr1-hv-con01.cern.ch:8500" + }, + "conditionDB": { + "url": "qcdb.cern.ch:8083" + } + }, + "tasks": { + "CellTaskEMCAL": { + "active": "true", + "taskName": "Cells", + "className": "o2::quality_control_modules::emcal::CellTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-cells:EMC/CELLS/0;emcal-triggerecords:EMC/CELLSTRGR/0" + }, + "taskParameters": { + "highMultiplicity": "true", + "MultiplicityRange": "8000", + "MultiplicityRangeSM" : "600", + "MultiplicityRangeSMThreshold": "200", + "TotalEnergyRangeDetector": "700", + "TotalEnergyRangeSM": "200", + "TotalEnergyRange": "2000" + } + }, + "ClusterTaskEMCAL": { + "active": "true", + "taskName": "Clusters", + "className": "o2::quality_control_modules::emcal::ClusterTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-cells:EMC/CELLS/0;emcal-triggerecords:EMC/CELLSTRGR/0" + }, + "taskParameters": { + "useInternalClusterizer": "true", + "bindingCellTriggerRecords": "emcal-triggerecords", + "calibrateCells": "false", + "clusterizerSeedThreshold": "0.5", + "clusterizerCellTreshold": "0.1", + "clusterizerDoGradientCut": "true", + "clusterizerGradientCut": "0.03", + "clusterizerMinTime": "-300.", + "clusterizerMaxTime": "300.", + "clusterizerMaxTimeDelta": "1000.", + "hasInvMassMesons": "true", + "mesonClustersRejectExotics": 1, + "mesonClusterMaxTime": "50.", + "MultiplicityRange":"400" + } + }, + "BCTaskEMCAL": { + "active": "true", + "taskName": "BCs", + "className": "o2::quality_control_modules::emcal::BCTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-triggers:EMC/CELLSTRGR;ctp-digits:CTP/DIGITS" + }, + "taskParameters": { + "AliasMB" : "CMTVXTSC" + } + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index b46e3f133..ae2ee0b63 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -157,8 +157,13 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_FT0:-}" ]] && QC_JSON_FT0=$O2DPG_ROOT/DATA/production/qc-async/ft0.json [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=$O2DPG_ROOT/DATA/production/qc-async/fv0.json [[ -z "${QC_JSON_FDD:-}" ]] && QC_JSON_FDD=$O2DPG_ROOT/DATA/production/qc-async/fdd.json - [[ -z "${QC_JSON_EMC:-}" ]] && QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc.json [[ -z "${QC_JSON_MID:-}" ]] && QC_JSON_MID=$O2DPG_ROOT/DATA/production/qc-async/mid.json + if [[ -z "${QC_JSON_EMC:-}" ]]; then + if [[ "$BEAMTYPE" == "PbPb" ]]; then + QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc_PbPb.json + else + QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc.json + fi if has_detector_qc MCH && [[ -z "${QC_JSON_MCH:-}" ]]; then add_QC_JSON MCH_DIGITS $O2DPG_ROOT/DATA/production/qc-async/mch-digits.json if has_processing_step "MCH_RECO"; then From 98155be586b57ddb97e939a57132ca2d41f26e2f Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Thu, 31 Aug 2023 14:47:35 +0200 Subject: [PATCH 0088/1239] [EMCAL-1037] Specifically request trigger records on subspec 0 In the asynchronous stage uncalibrated and calibrated cells and their corresponding trigger records are available on different subspecs. Only calibrated trigger records must be used. --- DATA/production/qc-async/emc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/qc-async/emc.json b/DATA/production/qc-async/emc.json index dfdfbb4c5..3a7d6fe8e 100644 --- a/DATA/production/qc-async/emc.json +++ b/DATA/production/qc-async/emc.json @@ -73,7 +73,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "emcal-triggers:EMC/CELLSTRGR;ctp-digits:CTP/DIGITS" + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS" }, "taskParameters": { } From c981a9de016eb76ebcf329dfa765a7a5047636dc Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Sun, 24 Sep 2023 08:35:25 +0200 Subject: [PATCH 0089/1239] Fix for EMC in qc-workflow --- DATA/production/qc-workflow.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index ae2ee0b63..8264c6ea3 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -163,6 +163,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc_PbPb.json else QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc.json + fi fi if has_detector_qc MCH && [[ -z "${QC_JSON_MCH:-}" ]]; then add_QC_JSON MCH_DIGITS $O2DPG_ROOT/DATA/production/qc-async/mch-digits.json From b0f8b5d2d493c91a7b4abdfc456818e64a343a7b Mon Sep 17 00:00:00 2001 From: Carlo Puggioni Date: Sun, 24 Sep 2023 13:01:45 +0200 Subject: [PATCH 0090/1239] QC Async ZDC REC and Postprocessing --- DATA/production/o2dpg_qc_postproc_workflow.py | 1 + DATA/production/qc-async/zdc.json | 57 +++++ DATA/production/qc-postproc-async/zdc.json | 227 ++++++++++++++++++ DATA/production/qc-workflow.sh | 1 + 4 files changed, 286 insertions(+) create mode 100644 DATA/production/qc-async/zdc.json create mode 100644 DATA/production/qc-postproc-async/zdc.json diff --git a/DATA/production/o2dpg_qc_postproc_workflow.py b/DATA/production/o2dpg_qc_postproc_workflow.py index 2edf58873..89b958a12 100755 --- a/DATA/production/o2dpg_qc_postproc_workflow.py +++ b/DATA/production/o2dpg_qc_postproc_workflow.py @@ -80,6 +80,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, periodSpec add_QC_postprocessing('example', 'json://${O2DPG_ROOT}/DATA/production/qc-postproc-async/example.json', needs=[], runSpecific=False, periodSpecific=False, passSpecific=True) add_QC_postprocessing('EMC', 'json://${O2DPG_ROOT}/DATA/production/qc-postproc-async/emc.json', needs=[], runSpecific=False, periodSpecific=True, passSpecific=True) add_QC_postprocessing('MCH', 'json://${O2DPG_ROOT}/DATA/production/qc-postproc-async/mch.json', needs=[], runSpecific=True, periodSpecific=True, passSpecific=True) + add_QC_postprocessing('ZDC', 'json://${O2DPG_ROOT}/DATA/production/qc-postproc-async/zdc.json', needs=[], runSpecific=True, periodSpecific=True, passSpecific=True) return stages diff --git a/DATA/production/qc-async/zdc.json b/DATA/production/qc-async/zdc.json new file mode 100644 index 000000000..88ca06a3e --- /dev/null +++ b/DATA/production/qc-async/zdc.json @@ -0,0 +1,57 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "alio2-cr1-hv-qcdb1.cern.ch:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2" + }, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "http://ali-consul.cern.ch:8500" + }, + "conditionDB": { + "url": "http://o2-ccdb.internal" + } + }, + "tasks": { + "ZDCRec": { + "active": "true", + "taskName": "Rec", + "className": "o2::quality_control_modules::zdc::ZDCRecDataTask", + "moduleName": "QcZDC", + "detectorName": "ZDC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "zdc-bcrec:ZDC/BCREC/0;zdc-energyrec:ZDC/ENERGY/0;zdc-tdcrec:ZDC/TDCDATA/0;zdc-inforec:ZDC/INFO/0" + }, + "taskParameters": { + "ADC": "3025;-100;12000", + "ADCH": "1100;-100;1000", + "TDCT": "2400;-25;25", + "TDCA": "2050;-0.5;4099.5", + "TDCAH": "2000;-0.5;3999.5", + "ADCSUMvsTC": "605;-100;12000;605;-100;12000", + "ADCvsTDCT": "240;-25;25;605;-100;12000", + "TDCDIFF": "240;-25;25;240;-25;25", + "TDCAvsTDCT": "480;-25;25;500;-1;3999", + "TDCAvsTDCA": "500;-1;3999;500;-1;3999", + "CENTR_ZNA": "200;-2;2;200;-2;2", + "CENTR_ZNC": "200;-2;2;200;-2;2", + "CENTR_ZPA": "2240;0;22.4", + "CENTR_ZPC": "2240;-22.4;0" + } + } + } + } +} diff --git a/DATA/production/qc-postproc-async/zdc.json b/DATA/production/qc-postproc-async/zdc.json new file mode 100644 index 000000000..a744a53d1 --- /dev/null +++ b/DATA/production/qc-postproc-async/zdc.json @@ -0,0 +1,227 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "alio2-cr1-hv-qcdb1.cern.ch:8083" + }, + "Activity": { + "number": "", + "provenance": "qc_async", + "periodName": "", + "passName": "apass0" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "http://o2-ccdb.internal" + }, + "postprocessing": { + "periodSeconds": "0.1" + }, + "infologger": { "": "Configuration of the Infologger (optional).", + "filterDiscardDebug": "true", "": "Set to 1 to discard debug and trace messages (default: false)", + "filterDiscardLevel": "6", "": "Message at this level or above are discarded (default: 21 - Trace)" + } + }, + "postprocessing": { + "RecPP": { + "active": "true", + "className": "o2::quality_control_modules::zdc::ZDCRecDataPostProcessing", + "moduleName": "QcZDC", + "detectorName": "ZDC", + "customization": [], + "dataSourcesADC": [ + { + "type": "repository", + "path": "ZDC/MO/Rec", + "names": [ + "ZNAC:h_ADC_ZNA_TC", + "ZNA1:h_ADC_ZNA_T1", + "ZNA2:h_ADC_ZNA_T2", + "ZNA3:h_ADC_ZNA_T3", + "ZNA4:h_ADC_ZNA_T4", + "ZNAS:h_ADC_ZNA_SUM", + "ZPAC:h_ADC_ZPA_TC", + "ZPA1:h_ADC_ZPA_T1", + "ZPA2:h_ADC_ZPA_T2", + "ZPA3:h_ADC_ZPA_T3", + "ZPA4:h_ADC_ZPA_T4", + "ZPAS:h_ADC_ZPA_SUM", + "ZEM1:h_ADC_ZEM1", + "ZEM2:h_ADC_ZEM2", + "ZNCC:h_ADC_ZNC_TC", + "ZNC1:h_ADC_ZNC_T1", + "ZNC2:h_ADC_ZNC_T2", + "ZNC3:h_ADC_ZNC_T3", + "ZNC4:h_ADC_ZNC_T4", + "ZNCS:h_ADC_ZNC_SUM", + "ZPCC:h_ADC_ZPC_TC", + "ZPC1:h_ADC_ZPC_T1", + "ZPC2:h_ADC_ZPC_T2", + "ZPC3:h_ADC_ZPC_T3", + "ZPC4:h_ADC_ZPC_T4", + "ZPCS:h_ADC_ZPC_SUM" + ] + } + ], + "dataSourcesTDC": [ + { + "type": "repository", + "path": "ZDC/MO/Rec", + "names": [ + "ZNAC:h_TDC_ZNA_TC_V", + "ZNAS:h_TDC_ZNA_SUM_V", + "ZPAC:h_TDC_ZPA_TC_V", + "ZPAS:h_TDC_ZPA_SUM_V", + "ZEM1:h_TDC_ZEM1_V", + "ZEM2:h_TDC_ZEM2_V", + "ZNCC:h_TDC_ZNC_TC_V", + "ZNCS:h_TDC_ZNC_SUM_V", + "ZPCC:h_TDC_ZPC_TC_V", + "ZPCS:h_TDC_ZPC_SUM_V" + ] + } + ], + "initTrigger": [ + "userorcontrol" + ], + "updateTrigger": [ + "foreachlatest:qcdb:ZDC/MO/Rec/h_ADC_ZNA_TC" + ], + "stopTrigger": [ + "userorcontrol" + ] + }, + "ZDCQuality": { + "active": "true", + "className": "o2::quality_control_modules::common::QualityTask", + "moduleName": "QualityControl", + "detectorName": "ZDC", + "qualityGroups": [ + { + "name": "global", + "title": "GLOBAL ZDC QUALITY", + "path": "ZDC/QO", + "ignoreQualitiesDetails": [ + "Null", + "Good", + "Medium", + "Bad" + ], + "inputObjects": [ + { + "name": "ZDCQuality/ZDCQuality", + "title": "ZDC Quality", + "messageBad": "Inform on-call immediately", + "messageMedium": "Add bookkeeping entry", + "messageGood": "All checks are OK", + "messageNull": "Some histograms are empty!!!" + } + ] + }, + { + "name": "details", + "title": "ZDC DETAILS", + "path": "ZDC/QO", + "ignoreQualitiesDetails": [], + "inputObjects": [ + { + "name": "QcZDCRecCheck", + "title": "Rec check" + } + ] + } + ], + "initTrigger": [ + "userorcontrol" + ], + "updateTrigger": [ + "foreachlatest:qcdb:ZDC/QO/ZDCQuality/ZDCQuality" + ], + "stopTrigger": [ + "userorcontrol" + ] + } + }, + "checks": { + "RecCheck": { + "active": "true", + "className": "o2::quality_control_modules::zdc::ZDCRecDataCheck", + "moduleName": "QcZDC", + "detectorName": "ZDC", + "policy": "OnAny", + "checkParameters": { + "ADC_ZNAC": "700;200;400", + "ADC_ZNA1": "260;100;200", + "ADC_ZNA2": "300;100;200", + "ADC_ZNA3": "170;100;200", + "ADC_ZNA4": "170;100;200", + "ADC_ZNAS": "700;200;400", + "ADC_ZPAC": "700;200;400", + "ADC_ZPA1": "60;50;100", + "ADC_ZPA2": "80;50;100", + "ADC_ZPA3": "245;50;100", + "ADC_ZPA4": "700;200;400", + "ADC_ZPAS": "850;200;400", + "ADC_ZEM1": "1230;200;400", + "ADC_ZEM2": "1275;200;400", + "ADC_ZNCC": "700;200;400", + "ADC_ZNC1": "130;100;200", + "ADC_ZNC2": "280;100;200", + "ADC_ZNC3": "130;100;200", + "ADC_ZNC4": "260;100;200", + "ADC_ZNCS": "600;200;400", + "ADC_ZPCC": "790;200;400", + "ADC_ZPC1": "770;200;400", + "ADC_ZPC2": "350;100;200", + "ADC_ZPC3": "97;50;100", + "ADC_ZPC4": "67;50;67", + "ADC_ZPCS": "1044;200;400", + "ADC_POS_MSG_X": "0.15", + "ADC_POS_MSG_Y": "0.92", + "TDC_ZNAC": "0.0;2.0;4.0", + "TDC_ZNAS": "0.0;2.0;4.0", + "TDC_ZPAC": "0.0;2.0;4.0", + "TDC_ZPAS": "0.0;2.0;4.0", + "TDC_ZEM1": "0.0;2.0;4.0", + "TDC_ZEM2": "0.0;2.0;4.0", + "TDC_ZNCC": "0.0;2.0;4.0", + "TDC_ZNCS": "0.0;2.0;4.0", + "TDC_ZPCC": "0.0;2.0;4.0", + "TDC_ZPCS": "0.0;2.0;4.0", + "TDC_POS_MSG_X": "0.01", + "TDC_POS_MSG_Y": "0.92" + }, + "dataSource": [ + { + "type": "PostProcessing", + "name": "RecPP", + "MOs": [ + "h_summary_ADC", "h_summary_TDC" + ] + } + ] + } + }, + "aggregators": { + "ZDCQuality": { + "active": "true", + "className": "o2::quality_control_modules::common::WorstOfAllAggregator", + "moduleName": "QcCommon", + "policy": "OnAll", + "detectorName": "ZDC", + "dataSource": [ + { + "type": "Check", + "name": "RecCheck" + } + ] + } + } + } +} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 8264c6ea3..90501a593 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -158,6 +158,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=$O2DPG_ROOT/DATA/production/qc-async/fv0.json [[ -z "${QC_JSON_FDD:-}" ]] && QC_JSON_FDD=$O2DPG_ROOT/DATA/production/qc-async/fdd.json [[ -z "${QC_JSON_MID:-}" ]] && QC_JSON_MID=$O2DPG_ROOT/DATA/production/qc-async/mid.json + [[ -z "${QC_JSON_ZDC:-}" ]] && has_processing_step ZDC_RECO && QC_JSON_ZDC=$O2DPG_ROOT/DATA/production/qc-async/zdc.json if [[ -z "${QC_JSON_EMC:-}" ]]; then if [[ "$BEAMTYPE" == "PbPb" ]]; then QC_JSON_EMC=$O2DPG_ROOT/DATA/production/qc-async/emc_PbPb.json From 8aae674d6a72d28d246c516e7e4be8250f9dab85 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 25 Sep 2023 09:32:35 +0200 Subject: [PATCH 0091/1239] dpl-workflow: Limit number of nodes in multiplicity computation to 230 to avoid downscaling --- DATA/production/workflow-multiplicities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index f71327797..b3d4de384 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -134,7 +134,7 @@ elif [[ $EPNPIPELINES != 0 ]]; then NTRDTRKTHREADS=2 ITSTRK_THREADS=2 ITSTPC_THREADS=2 - RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? $RECO_NUM_NODES_WORKFLOW : 15) * ($NUMAGPUIDS != 0 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains + RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? ($RECO_NUM_NODES_WORKFLOW < 230 ? $RECO_NUM_NODES_WORKFLOW : 230) : 15) * ($NUMAGPUIDS != 0 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains # Tuned multiplicities for sync pp / Pb-Pb processing if [[ $BEAMTYPE == "pp" ]]; then N_ITSRAWDEC=$(math_max $((6 * $EPNPIPELINES * $NGPUS / 4)) 1) From e2b5dbc147f9b6666ca22050ae171768e066ed24 Mon Sep 17 00:00:00 2001 From: aimeric-landou <46970521+aimeric-landou@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:10:24 +0200 Subject: [PATCH 0092/1239] Add HF injected jet-jet sim scripts (#1236) * Add HF injected jet-jet sim scripts --- MC/run/PWGGAJE/run_jets_HF_bbbar.sh | 57 +++++++++++++++++++++++++++++ MC/run/PWGGAJE/run_jets_HF_ccbar.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 MC/run/PWGGAJE/run_jets_HF_bbbar.sh create mode 100644 MC/run/PWGGAJE/run_jets_HF_ccbar.sh diff --git a/MC/run/PWGGAJE/run_jets_HF_bbbar.sh b/MC/run/PWGGAJE/run_jets_HF_bbbar.sh new file mode 100644 index 000000000..be00d7fe9 --- /dev/null +++ b/MC/run/PWGGAJE/run_jets_HF_bbbar.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Generate jet-jet events with ccbar HF injected, Pythia8 in a pre-defined pt hard bin and weighted. +# Execute: ./run_jets_HF_bbbar.sh + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + + +# ----------- START ACTUAL JOB ----------------------------- + +RNDSEED=${RNDSEED:-0} # [default = 0] time-based random seed + +NSIGEVENTS=${NSIGEVENTS:-10} +NTIMEFRAMES=${NTIMEFRAMES:-1} +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" #"PIPE ITS TPC EMCAL" +CONFIG_ENERGY=${CONFIG_ENERGY:-13600.0} +SIMENGINE=${SIMENGINE:-TGeant4} +WEIGHTPOW=${WEIGHTPOW:-6.0} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +# Default for weighted productions +PTHATMIN=${PTHATMIN:-5.0} +PTHATMAX=${PTHATMAX:-300.0} + +# Define the pt hat bin arrays +pthatbin_loweredges=(0 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235) +pthatbin_higheredges=( 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235 -1) + +# Recover environmental vars for pt binning +#PTHATBIN=${PTHATBIN:-1} + +if [ -z "$PTHATBIN" ]; then + echo "Open Pt-hat range set" +else + PTHATMIN=${pthatbin_loweredges[$PTHATBIN]} + PTHATMAX=${pthatbin_higheredges[$PTHATBIN]} +fi + + +#ccbar filter +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${CONFIG_ENERGY} -col pp -gen external -proc "jets" \ + -ptHatMin ${PTHATMIN} -ptHatMax ${PTHATMAX} \ + -tf ${NTIMEFRAMES} -ns ${NSIGEVENTS} -e ${SIMENGINE} \ + -j ${NWORKERS} -mod "--skipModules ZDC" \ + -interactionRate 500000 -confKey "Diamond.width[2]=6." ${SEED} \ + -ini $O2DPG_ROOT/MC/config/PWGHF/ini/GeneratorHFTrigger_bbbar.ini \ + -weightPow ${WEIGHTPOW} + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 \ No newline at end of file diff --git a/MC/run/PWGGAJE/run_jets_HF_ccbar.sh b/MC/run/PWGGAJE/run_jets_HF_ccbar.sh new file mode 100644 index 000000000..3149f5092 --- /dev/null +++ b/MC/run/PWGGAJE/run_jets_HF_ccbar.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Generate jet-jet events with ccbar HF injected, Pythia8 in a pre-defined pt hard bin and weighted. +# Execute: ./run_jets_HF_ccbar.sh + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + + +# ----------- START ACTUAL JOB ----------------------------- + +RNDSEED=${RNDSEED:-0} # [default = 0] time-based random seed + +NSIGEVENTS=${NSIGEVENTS:-10} +NTIMEFRAMES=${NTIMEFRAMES:-1} +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" #"PIPE ITS TPC EMCAL" +CONFIG_ENERGY=${CONFIG_ENERGY:-13600.0} +SIMENGINE=${SIMENGINE:-TGeant4} +WEIGHTPOW=${WEIGHTPOW:-6.0} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +# Default for weighted productions +PTHATMIN=${PTHATMIN:-5.0} +PTHATMAX=${PTHATMAX:-300.0} + +# Define the pt hat bin arrays +pthatbin_loweredges=(0 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235) +pthatbin_higheredges=( 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235 -1) + +# Recover environmental vars for pt binning +#PTHATBIN=${PTHATBIN:-1} + +if [ -z "$PTHATBIN" ]; then + echo "Open Pt-hat range set" +else + PTHATMIN=${pthatbin_loweredges[$PTHATBIN]} + PTHATMAX=${pthatbin_higheredges[$PTHATBIN]} +fi + + +#ccbar filter +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${CONFIG_ENERGY} -col pp -gen external -proc "jets" \ + -ptHatMin ${PTHATMIN} -ptHatMax ${PTHATMAX} \ + -tf ${NTIMEFRAMES} -ns ${NSIGEVENTS} -e ${SIMENGINE} \ + -j ${NWORKERS} -mod "--skipModules ZDC" \ + -interactionRate 500000 -confKey "Diamond.width[2]=6." ${SEED} \ + -ini $O2DPG_ROOT/MC/config/PWGHF/ini/GeneratorHFTrigger_ccbar.ini \ + -weightPow ${WEIGHTPOW} + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 \ No newline at end of file From a36e26043b15928a6fba9fc29b7545487949f528 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Fri, 22 Sep 2023 14:03:20 +0200 Subject: [PATCH 0093/1239] [RelVal] Updates * accept any number of histograms for overlay * make 1D overlay plots nicer to look at * automatic log10 scale if integrals differ by more than 2 orders of magnitude * unified relative y-ranges for nominal and ratio plot * choose different line colors * unify label and title sizes * re-position legends * choose x-range such that everything is seen (sometimes the x-axis range is huge while entries are only in some of the first bins and nothing could be seen) * more ratio plot directly underneath nominal plot and share x-axis * indicate in overlay plots if something is empty * always copy overlay plots with inspect command * copy those overlays that satisfy selection criteria * add brief QC section to README --- RelVal/PlotOverlays.C | 225 ++++++++++++++++++++++++----- RelVal/README.md | 36 ++++- RelVal/o2dpg_release_validation.py | 76 ++++++---- 3 files changed, 266 insertions(+), 71 deletions(-) diff --git a/RelVal/PlotOverlays.C b/RelVal/PlotOverlays.C index 0e43c7b46..6ac74d090 100644 --- a/RelVal/PlotOverlays.C +++ b/RelVal/PlotOverlays.C @@ -2,61 +2,195 @@ #include #include + + +void findRangeNotEmpty1D(TH1* h, double& min, double& max) +{ + auto axis = h->GetXaxis(); + min = axis->GetBinLowEdge(1); + max = axis->GetBinUpEdge(axis->GetNbins()); + for (int i = 1; i <= axis->GetNbins(); i++) { + if (h->GetBinContent(i) != 0) { + min = axis->GetBinLowEdge(i); + break; + } + } + for (int i = axis->GetNbins(); i >= 1 ; i--) { + if (h->GetBinContent(i) != 0) { + max = axis->GetBinUpEdge(i); + break; + } + } +} + + +TH1* makeFrameFromHistograms(TPad& pad, std::vector const& histograms, bool& shouldBeLog) +{ + // make a frame to fit all histograms + // propose log10 scale in case some integrals differ by more than 2 orders of magnitude + auto integralRef = histograms[0]->Integral(); + shouldBeLog = false; + auto minY = histograms[0]->GetMinimum(0); + double maxY = histograms[0]->GetMaximum(); + + double minX; + double maxX; + findRangeNotEmpty1D(histograms[0], minX, maxX); + + // find minima and maxima + for (int i = 1; i < histograms.size(); i++) { + minY = std::min(histograms[i]->GetMinimum(0), minY); + maxY = std::max(histograms[i]->GetMaximum(), maxY); + + double minXNext; + double maxXNext; + findRangeNotEmpty1D(histograms[i], minXNext, maxXNext); + minX = std::min(minX, minXNext); + maxX = std::max(maxX, maxXNext); + + auto integral = histograms[i]->Integral(); + if ((integralRef > 0 && integral / integralRef > 100) || (integral > 0 && integralRef / integral > 100)) { + // decide whether to do a log plot + shouldBeLog = true; + } + } + + // finalise the y-axis limits + if (shouldBeLog) { + auto margin = std::log10(maxY / minY); + minY = minY / std::pow(10, margin * 0.1); + maxY = maxY * std::pow(10, margin * 0.3);; + } else { + auto margin = 0.1 * (maxY - minY); + maxY += 3 * margin; + minY -= std::max(0., margin); + } + + if (histograms[0]->GetXaxis()->IsAlphanumeric()) { + auto alphanumericFrame = (TH1*)histograms[0]->Clone(); + alphanumericFrame->Reset("ICEMS"); + return alphanumericFrame; + } + + return pad.DrawFrame(minX, minY, maxX, maxY); +} + // overlay 1D histograms -void overlay1D(std::vector hVec, std::vector labelVec, TLegend* legend, std::string const& outputDir) +void overlay1D(std::vector hVec, std::vector labelVec, TLegend* additionalLegend, std::string const& outputDir) { TCanvas c("overlay", "", 800, 800); c.cd(); + TPad nominalPad("nominalPad", "nominalPad", 0, 0.3, 1., 1.); + nominalPad.SetBottomMargin(0); + TPad ratioPad("ratioPad", "ratioPad", 0, 0.05, 1. ,0.32); + ratioPad.SetTopMargin(0); + ratioPad.SetBottomMargin(0.2); - TPad lower_pad("lower_pad","lower_pad",0,0.05,1.,0.3); - TPad upper_pad("upper_pad","upper_pad",0,0.25,1.,1.); - upper_pad.Draw(); - lower_pad.Draw(); + nominalPad.Draw(); + ratioPad.Draw(); - const int colors[6]={kRed,kBlue,kGreen,kMagenta,kCyan,kOrange}; //TODO what if more then 6 histograms - const int linestyles[6]={1,10,2,9,8,7}; //TODO what if more then 6 histograms + const int colors[7] = {kRed + 2, kBlue - 4, kGreen + 3, kMagenta + 1, kCyan + 2, kOrange + 5, kYellow - 6}; + const int linestyles[6] = {1, 10, 2, 9, 8, 7}; - TLegend legendOverlay(0.65, 0.8, 0.9, 0.9); + TLegend legendOverlay(0.65, 0.7, 0.9, 0.9); legendOverlay.SetFillStyle(0); - int counter = 0; - for (auto h : hVec){ - upper_pad.cd(); + legendOverlay.SetBorderSize(0); + + bool logY{}; + nominalPad.cd(); + auto frame = makeFrameFromHistograms(nominalPad, hVec, logY); + frame->SetTitle(hVec[0]->GetTitle()); + auto yAxis = frame->GetYaxis(); + yAxis->ChangeLabel(1, -1, -1, -1, -1, -1, " "); + yAxis->SetTitleFont(43); + yAxis->SetTitleSize(20); + yAxis->SetLabelFont(43); + yAxis->SetLabelSize(20); + yAxis->SetTitle(hVec[0]->GetYaxis()->GetTitle()); + auto xAxis = frame->GetXaxis(); + xAxis->SetLabelFont(43); + xAxis->SetLabelSize(0); + + + std::vector ratios; + + std::string emptyText; + for (int i = 0; i < hVec.size(); i++) { + auto& h = hVec[i]; + h->SetStats(0); - h->SetLineStyle(linestyles[counter]); + h->SetLineStyle(linestyles[i % 6]); h->SetLineWidth(1); - h->SetLineColor(colors[counter]); - TH1F* hClone = (TH1F*)h->Clone(); + h->SetLineColor(colors[i % 7]); - h->GetXaxis()->SetLabelSize(0.); - h->GetXaxis()->SetLabelOffset(999); - h->GetYaxis()->SetLabelSize(0.05); + if (i > 0) { + // no ratio for the first histogram (which would simply be 1) + TH1* hRatio = (TH1*)h->Clone(); + hRatio->Divide(h, hVec[0], 1.0, 1.0, "B"); // error option? + ratios.push_back(hRatio); + } - legendOverlay.AddEntry(h, labelVec[counter].c_str()); + legendOverlay.AddEntry(h, labelVec[i].c_str()); h->Draw("same E hist"); - - lower_pad.cd(); - hClone->Divide(h,hVec[0],1.0,1.0,"B"); // error option? - hClone->SetTitle(""); - hClone->SetLineStyle(1); - hClone->GetYaxis()->SetRangeUser(0.,10.); - hClone->GetYaxis()->SetLabelSize(0.125); - hClone->GetXaxis()->SetLabelSize(0.125); - if (counter>0){ - hClone->Draw("same E1"); - } - else { - hClone->Draw("same"); + if (h->GetEntries() == 0) { + emptyText += labelVec[i] + ", "; } - counter++; } - upper_pad.cd(); - if (legend){ - legend->Draw("same"); + + if (logY) { + nominalPad.SetLogy(); + } + + if (additionalLegend) { + additionalLegend->SetBorderSize(0); + additionalLegend->SetFillStyle(0); + // To reposition the legend we need to: Draw, Update, set new coordinates, Modified + additionalLegend->Draw("same"); + nominalPad.Update(); + additionalLegend->SetX1NDC(0.15); + additionalLegend->SetY1NDC(0.7); + additionalLegend->SetX2NDC(0.4); + additionalLegend->SetY2NDC(0.9); + nominalPad.Modified(); } legendOverlay.Draw("same"); + if (!emptyText.empty()) { + emptyText.pop_back(); + emptyText.pop_back(); + emptyText = std::string("EMPTY: ") + emptyText; + TText *t1 = new TText(0.2, 0.5, emptyText.c_str()); + t1->SetNDC(); + t1->Draw(); + } + + ratioPad.cd(); + frame = makeFrameFromHistograms(ratioPad, ratios, logY); + yAxis = frame->GetYaxis(); + yAxis->SetTitleFont(43); + yAxis->SetTitleSize(20); + yAxis->SetLabelFont(43); + yAxis->SetLabelSize(20); + yAxis->SetTitle("ratio"); + + xAxis = frame->GetXaxis(); + xAxis->SetTitleFont(43); + xAxis->SetTitleSize(20); + xAxis->SetLabelFont(43); + xAxis->SetLabelSize(20); + xAxis->SetTitle(hVec[0]->GetXaxis()->GetTitle()); + + for (int i = 0; i < ratios.size(); i++) { + auto& h = ratios[i]; + h->Draw("same"); + } + + if (logY) { + ratioPad.SetLogy(); + } + auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; c.SaveAs(savePath.c_str()); c.Close(); @@ -71,14 +205,20 @@ void overlay2D(std::vector hVec1, std::vector labelVec, TLege } int nHistos = hVec.size(); - TCanvas c("overlay", "", 2400, 800*(nHistos-1)); - c.Divide(3, nHistos-1); + TCanvas c("overlay", "", 2400, 800 * (nHistos-1)); + c.Divide(3, nHistos - 1); c.cd(1); hVec[0]->SetTitle(hVec[0]->GetTitle() + TString("(" + labelVec[0] + ")")); hVec[0]->SetStats(0); hVec[0]->Draw("colz"); - for (int i = 1; iGetEntries() == 0) { + TText *t1 = new TText(0.5, 0.5, "EMPTY"); + t1->SetNDC(); + t1->Draw(); + } + + for (int i = 1; i < nHistos; i++){ auto hDiv = (TH2*)hVec[i]->Clone(Form("%s_ratio", hVec[i]->GetName())); hDiv->SetTitle(hVec[i]->GetTitle() + TString("(" + labelVec[i] + "/"+labelVec[0]+")")); hDiv->SetStats(0); @@ -86,10 +226,15 @@ void overlay2D(std::vector hVec1, std::vector labelVec, TLege hVec[i]->SetTitle(hVec[i]->GetTitle() + TString("(" + labelVec[i] + ")")); hVec[i]->SetStats(0); - c.cd(i*3-1); + c.cd(i * 3 - 1); hVec[i]->Draw("colz"); + if (hVec[i]->GetEntries() == 0) { + TText *t1 = new TText(0.5, 0.5, "EMPTY"); + t1->SetNDC(); + t1->Draw(); + } - c.cd(i*3); + c.cd(i * 3); hDiv->Draw("colz"); } diff --git a/RelVal/README.md b/RelVal/README.md index 089c8fcb6..e50a897fc 100644 --- a/RelVal/README.md +++ b/RelVal/README.md @@ -42,7 +42,7 @@ The [Python script](o2dpg_release_validation.py) is the entrypoint of the RelVal The full help message of this script can be seen by typing ```bash -python o2dpg_release_validation.py [] --help +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py [] --help ``` The wrapper includes 3 different sub-commands for now 1. `rel-val` to steer the RelVal, @@ -82,11 +82,14 @@ ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ] [--exclude-patterns ] \ [--enable-metric ] [--disable-metric ] \ [--interpretations ] \ - [--critical ] + [--critical ] \ + [--output|-o ] ``` All of those options, except for `--include-patterns` and `--exclude-patterns` also work with the `rel-val` command. The output will by default be written to `rel_val_inspect`. All plots which are produced by the `rel-val` command are produced again for a potential given sub-set depending on the given options. Only the overlay plots are not produced again. +**NOTE** that with `inspect` the original overlay plots satisfying your selection criteria (e.g. `--include-patters` or `--interpretations`) are also copied over to the target directory. + **Other additional optional arguments** * `--use-values-as-thresholds []`: By passing a set of summaries that where produced from `rel-val`, the computed metric values can be used as **new** thresholds. To decide how to combine the values for multiple metrics referring to the same object, the option `--combine-thresholds mean|extreme` can be used. Also, an additional relative margin can be added for each metric with `--margin-threshold `; this argument must be repeated for if it should be used for multiple metrics. * `--regions []`: This computes means and standard deviations for each metric from previously computed values. The corresponding test is passed, if the value lies around the mean within the standard deviations. The deviation from the mean is also given as number-of-sigmas in the summary grid. @@ -106,7 +109,7 @@ ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py print --metric-names To convert the final output to something that can be digested by InfluxDB, use ```bash -python ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py influx --dir [--tags k1=v1 k2=v2 ...] [--table-name ] +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py influx --dir [--tags k1=v1 k2=v2 ...] [--table-name ] ``` When the `--tags` argument is specified, these are injected as TAGS for InfluxDB in addition. The table name can also be specified explicitly; if not given, it defaults to `O2DPG_MC_ReleaseValidation`. @@ -118,8 +121,33 @@ There is an ongoing effort to unify the names of QC objects inside MC and data Q MC QC objects are usually distributed over multiple files while those from data are all contained in one single file. It is possible to directly compare them with ```bash -python ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${MC_PRODUCTION}/QC/*.root -j ${DATA_PRODUCTION}/QC.root [--include-dirs ] +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${MC_PRODUCTION}/QC/*.root -j ${DATA_PRODUCTION}/QC.root [--include-dirs ] +``` + +## Run for QC +This is a simple guide to run RelVal for QC. + +### If you are interested in all QC plots +To have everything and to use this as a starting point for deeper inspections, first run +```bash +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i QC_file_1.root -j QC_file_2.root -o rel_val_all [--labels meaningfulLabel1 meaningfulLabel2] +``` +Now, there is of course a lot but from now on you are fully flexible. + +In order to get some insight into a specific detector, say ITS, run +```bash +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path rel_val_all --include-patterns "^ITS_" -o rel_val_ITS +``` +This will only print pie charts and summaries for ITS and also copies all overlay plots related to ITS to your target directory `rel_val_ITS`. + +The `inspect` command is much faster now since no new plots are generated and metrics do not have to be recomputed. It simply filters the results according to your criteria. However, what can be re-evaluated are the computed values against new thresholds. + +### If you are only interested in some ROOT sub-directories to begin with +If you only want to study for instance the ITS and CPV and there is no interest at this point to study any other detector, run +```bash +${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i QC_file_1.root -j QC_file_2.root -o rel_val_all --include-dirs ITS CPV [--labels meaningfulLabel1 meaningfulLabel2] ``` +From here on, you can use the `inspect` command as usual. But there will never be detectors other than ITS and CPV. ## Expert section diff --git a/RelVal/o2dpg_release_validation.py b/RelVal/o2dpg_release_validation.py index e4a0be6be..adccdc982 100755 --- a/RelVal/o2dpg_release_validation.py +++ b/RelVal/o2dpg_release_validation.py @@ -11,9 +11,9 @@ import sys import argparse import importlib.util -from os import environ, makedirs, remove +from os import environ, makedirs, remove, rename from os.path import join, abspath, exists, dirname, basename, isfile -from shutil import copy +from shutil import copy, rmtree # make sure O2DPG + O2 is loaded O2DPG_ROOT=environ.get('O2DPG_ROOT') @@ -51,6 +51,45 @@ gROOT.SetBatch() +def copy_overlays(rel_val, input_dir, output_dir): + """ + copy overlay plots in this summary from the input directory to the output directory + """ + input_dir = abspath(input_dir) + output_dir = abspath(output_dir) + + if not exists(input_dir): + print(f"ERROR: Input directory {input_dir} does not exist") + return 1 + + inOutSame = input_dir == output_dir + + input_dir_new = input_dir + "_tmp" + if inOutSame: + # move input directory + rename(input_dir, input_dir_new) + input_dir = input_dir_new + + if not exists(output_dir): + makedirs(output_dir) + + object_names, _ = rel_val.get_result_per_metric_and_test() + object_names = list(set(object_names)) + + ret = 0 + for object_name in object_names: + filename=join(input_dir, f"{object_name}.png") + if exists(filename): + copy(filename, output_dir) + else: + print(f"File {filename} not found.") + ret = 1 + + if inOutSame: + rmtree(input_dir) + + return ret + def metrics_from_root(): """ @@ -107,11 +146,11 @@ def extract(input_filenames, target_filename, include_file_directories="", add_i target_filename = basename(target_filename) log_file_name = join(cwd, f"{target_filename}_extract_and_flatten.log") - print(f"Extraction of files\n{','.join(input_filenames)}") + print("Extraction of files") for f in input_filenames: f = abspath(f) - print(f) + print(f" {f}") cmd = f"\\(\\\"{f}\\\",\\\"{target_filename}\\\",\\\"{reference_extracted}\\\",\\\"{include_file_directories}\\\"\\)" cmd = f"root -l -b -q {ROOT_MACRO_EXTRACT}{cmd}" ret = run_macro(cmd, log_file_name, cwd) @@ -286,8 +325,10 @@ def interpret_results(result, metric): makedirs(args.output) need_apply = False + is_inspect = False if hasattr(args, "json_path"): # this comes from the inspect command + is_inspect = True json_path = get_summary_path(args.json_path) annotations = None include_patterns, exclude_patterns = (args.include_patterns, args.exclude_patterns) @@ -322,6 +363,9 @@ def filter_on_interpretations(result): # if this comes from inspecting, there will be the annotations from the rel-val before that ==> re-write it rel_val.write(join(args.output, "Summary.json"), annotations=annotations or rel_val.annotations[0]) + if is_inspect: + copy_overlays(rel_val, join(dirname(json_path), "overlayPlots"), join(args.output, "overlayPlots")) + if not args.no_plot: # plot various different figures for user inspection plot_pie_charts(rel_val, variables.REL_VAL_SEVERITIES, variables.REL_VAL_SEVERITY_COLOR_MAP, args.output) @@ -332,23 +376,6 @@ def filter_on_interpretations(result): return 0 -def copy_overlays(path, output_dir,summary): - """ - copy overlay plots in this summary from the input directory to the output directory - """ - path = join(dirname(path),"overlayPlots") - output_dir = join(output_dir,"overlayPlots") - if not exists(output_dir): - makedirs(output_dir) - for histoname in summary: - filename=join(path,histoname+".png") - if exists(filename): - copy(filename,output_dir) - else: - print(f"File {filename} not found.") - return 0 - - def compare(args): """ Compare 2 RelVal outputs with one another @@ -393,9 +420,6 @@ def compare(args): s += f", {in_common}, {only_in1}, {only_in2}" print(s) - - - # plot comparison of values and thresholds of both RelVals per test if args.plot: if not exists(output_dir): @@ -518,16 +542,14 @@ def print_header(): PARSER = argparse.ArgumentParser(description='Wrapping ReleaseValidation macro') SUB_PARSERS = PARSER.add_subparsers(dest="command") REL_VAL_PARSER = SUB_PARSERS.add_parser("rel-val", parents=[COMMON_FILE_PARSER, COMMON_METRIC_PARSER, COMMON_THRESHOLD_PARSER, COMMON_FLAGS_PARSER, COMMON_VERBOSITY_PARSER]) -REL_VAL_PARSER.add_argument("--include-dirs", dest="include_dirs", nargs="*", help="only include directories; note that each pattern is assumed to start in the top-directory (at the moment no regex or *)") +REL_VAL_PARSER.add_argument("--include-dirs", dest="include_dirs", nargs="*", help="only include desired directories inside ROOT file; note that each pattern is assumed to start in the top-directory (at the moment no regex or *)") REL_VAL_PARSER.add_argument("--add", action="store_true", help="If given and there is already a RelVal in the output directory, extracted objects will be added to the existing ones") REL_VAL_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val") -REL_VAL_PARSER.add_argument("--copy-overlays", dest="copy_overlays", action="store_true", help="Copy overlay plots that meet the filter criteria to output directory") REL_VAL_PARSER.set_defaults(func=rel_val) INSPECT_PARSER = SUB_PARSERS.add_parser("inspect", parents=[COMMON_THRESHOLD_PARSER, COMMON_METRIC_PARSER, COMMON_PATTERN_PARSER, COMMON_FLAGS_PARSER, COMMON_VERBOSITY_PARSER]) INSPECT_PARSER.add_argument("--path", dest="json_path", help="either complete file path to a Summary.json or directory where one of the former is expected to be", required=True) INSPECT_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val_inspect") -INSPECT_PARSER.add_argument("--copy-overlays", dest="copy_overlays", action="store_true", help="Copy overlay plots that meet the filter criteria to output directory") INSPECT_PARSER.set_defaults(func=rel_val) COMPARE_PARSER = SUB_PARSERS.add_parser("compare", parents=[COMMON_FILE_PARSER, COMMON_PATTERN_PARSER, COMMON_METRIC_PARSER, COMMON_VERBOSITY_PARSER, COMMON_FLAGS_PARSER]) From d992550739615c4efd4e7efdb429242b9fd1ab5b Mon Sep 17 00:00:00 2001 From: wiechula Date: Tue, 26 Sep 2023 16:42:29 +0200 Subject: [PATCH 0094/1239] Add modifications to activate TPC DCAr plots * proper ccdb * grpGeom handler --- DATA/production/qc-async/tpc.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index 6f7e06409..af807e718 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -23,7 +23,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -91,6 +91,16 @@ "cutAbsEta": "1.", "cutMinNCluster": "60", "cutMindEdxTot": "20." + }, + "grpGeomRequest" : { + "geomRequest": "None", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "true", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" } } } From fcc4d13ce92564ff416a2bc95719ad3142a61d16 Mon Sep 17 00:00:00 2001 From: noferini Date: Tue, 19 Sep 2023 19:24:30 +0200 Subject: [PATCH 0095/1239] use FT0 in TOF calibs --- DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh | 2 +- DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh | 2 +- .../configurations/2022/LHC22f/apass1/setenv_extra.sh | 2 +- .../configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh | 2 +- .../configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh | 2 +- .../production/configurations/2022/extractCalib/setenv_extra.sh | 2 +- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh b/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh index b6c1fdf96..45c4885ee 100644 --- a/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh +++ b/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh @@ -65,7 +65,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh b/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh index b6c1fdf96..45c4885ee 100644 --- a/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh +++ b/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh @@ -65,7 +65,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh index 648e9a61b..ca64edd66 100644 --- a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh @@ -349,7 +349,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$VDRIFTPARAMOPTION;$TRACKTUNETPCINNER;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh b/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh index 42edfbcf5..c70f06a51 100644 --- a/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh @@ -95,7 +95,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="TPCGasParam.DriftV=$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="TPCGasParam.DriftV=$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh b/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh index 095511c42..b2c79141f 100644 --- a/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh +++ b/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh @@ -111,7 +111,7 @@ if [[ $WORKFLOW_DETECTORS =~ (^|,)"FT0"(,|$) ]] ; then fi # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$ITSEXTRAERR;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/extractCalib/setenv_extra.sh b/DATA/production/configurations/2022/extractCalib/setenv_extra.sh index 10bf1543c..a4fac7e44 100644 --- a/DATA/production/configurations/2022/extractCalib/setenv_extra.sh +++ b/DATA/production/configurations/2022/extractCalib/setenv_extra.sh @@ -69,7 +69,7 @@ if [[ $WORKFLOW_DETECTORS =~ (^|,)"FT0"(,|$) ]] ; then fi # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$ITSEXTRAERR;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 8a9c2bdb3..ad5ced062 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -400,7 +400,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching From 0d93a482fd6fa95e9647fc7014ff9fa064b50137 Mon Sep 17 00:00:00 2001 From: Lucamicheletti93 Date: Mon, 25 Sep 2023 10:29:19 +0200 Subject: [PATCH 0096/1239] Adding script for Nuclei at Fwd & rapidity range in the .gun file --- .../PWGLF/ini/GeneratorLFNucleiFwdppGap.ini | 6 ++++ .../PWGLF/pythia8/generator/hypernuclei.gun | 12 +++---- .../PWGLF/pythia8/generator/nuclei_fwd.gun | 2 ++ ...generator_pythia8_longlived_gaptriggered.C | 28 ++++++++++------ MC/run/PWGLF/run_NucleiFwdInjectedGap.sh | 33 +++++++++++++++++++ 5 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 MC/config/PWGLF/ini/GeneratorLFNucleiFwdppGap.ini create mode 100644 MC/config/PWGLF/pythia8/generator/nuclei_fwd.gun create mode 100755 MC/run/PWGLF/run_NucleiFwdInjectedGap.sh diff --git a/MC/config/PWGLF/ini/GeneratorLFNucleiFwdppGap.ini b/MC/config/PWGLF/ini/GeneratorLFNucleiFwdppGap.ini new file mode 100644 index 000000000..7c14f1cff --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFNucleiFwdppGap.ini @@ -0,0 +1,6 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/nuclei_fwd.gun", 5) + +[GeneratorPythia8] +config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun index 154cb401e..09e6ef26e 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun @@ -1,6 +1,6 @@ -# PDG N ptMin ptMax -1000010020 1 0.2 6 -1000010030 1 0.2 6 -1000020030 1 0.2 6 -1000020040 1 0.2 6 -1010010030 1 0.2 6 +# PDG N ptMin ptMax yMin yMax +1000010020 1 0.2 6 -1 1 +1000010030 1 0.2 6 -1 1 +1000020030 1 0.2 6 -1 1 +1000020040 1 0.2 6 -1 1 +1010010030 1 0.2 6 -1 1 diff --git a/MC/config/PWGLF/pythia8/generator/nuclei_fwd.gun b/MC/config/PWGLF/pythia8/generator/nuclei_fwd.gun new file mode 100644 index 000000000..d815845c4 --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/nuclei_fwd.gun @@ -0,0 +1,2 @@ +# PDG N ptMin ptMax yMin yMax +1000020030 10 0. 10. -4. -2.5 \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C index c34411fb7..ff67e41d9 100644 --- a/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +++ b/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C @@ -21,12 +21,13 @@ class GeneratorPythia8LongLivedGapTriggered : public o2::eventgen::GeneratorPyth { public: /// Constructor - GeneratorPythia8LongLivedGapTriggered(std::vector input_pdg, int input_trigger_ratio = 1, int n_injected = 1, float pt_min = 1, float pt_max = 10) + GeneratorPythia8LongLivedGapTriggered(std::vector input_pdg, int input_trigger_ratio = 1, int n_injected = 1, float pt_min = 1, float pt_max = 10, float y_min = -1, float y_max = 1) { mPdg = input_pdg; setNinjected(n_injected); mInverseTriggerRatio = input_trigger_ratio; setPt(pt_min, pt_max); + setY(y_min, y_max); mMass = getMass(input_pdg); mGeneratedEvents = 0; mAlternatingPDGsign = true; @@ -42,18 +43,22 @@ public: int pdg = 0; unsigned long n_inj = 0; float pt_min = 0.; - float pt_max = 0; + float pt_max = 0.; + float y_min = 0.; + float y_max = 0.; if (!config_file.is_open()) { LOGF(fatal, "File %s cannot be opened.", expanded_file_name); } std::getline(config_file, header); // skip first line - while (config_file >> pdg >> n_inj >> pt_min >> pt_max) + while (config_file >> pdg >> n_inj >> pt_min >> pt_max >> y_min >> y_max) { mPdg.push_back(pdg); mNinjected.push_back(n_inj); mPtMin.push_back(pt_min); mPtMax.push_back(pt_max); + mYmin.push_back(y_min); + mYmax.push_back(y_max); } config_file.close(); mInverseTriggerRatio = input_trigger_ratio; @@ -81,8 +86,11 @@ public: /// Set rapidity void setY(float y_min, float y_max) { - mYmin = y_min; - mYmax = y_max; + for (auto part : mPdg) + { + mYmin.push_back(y_min); + mYmax.push_back(y_max); + } } /// Set pseudorapidity @@ -130,7 +138,7 @@ public: for (int i = 0; i < mNinjected[injectionIndex]; ++i) { const double pt = gRandom->Uniform(mPtMin[injectionIndex], mPtMax[injectionIndex]); - const double rapidity = gRandom->Uniform(mYmin, mYmax); + const double rapidity = gRandom->Uniform(mYmin[injectionIndex], mYmax[injectionIndex]); const double phi = gRandom->Uniform(0, TMath::TwoPi()); const double px{pt * std::cos(phi)}; const double py{pt * std::sin(phi)}; @@ -157,8 +165,8 @@ private: std::vector mPtMin; /// minimum transverse momentum for generated particles std::vector mPtMax; /// maximum transverse momentum for generated particles - double mYmin = -1.; /// minimum rapidity for generated particles - double mYmax = +1.; /// maximum rapidity for generated particles + std::vector mYmin; /// minimum rapidity for generated particles + std::vector mYmax; /// maximum rapidity for generated particles bool mAlternatingPDGsign = true; /// bool to randomize the PDG code of the core particle @@ -170,9 +178,9 @@ private: }; ///___________________________________________________________ -FairGenerator *generateLongLivedGapTriggered(std::vector mPdg, int input_trigger_ratio, int n_injected = 1, float pt_min = 1, float pt_max = 10, bool alternate_sign = true) +FairGenerator *generateLongLivedGapTriggered(std::vector mPdg, int input_trigger_ratio, int n_injected = 1, float pt_min = 1, float pt_max = 10, float y_min = -1, float y_max = 1, bool alternate_sign = true) { - auto myGen = new GeneratorPythia8LongLivedGapTriggered(mPdg, input_trigger_ratio, n_injected, pt_min, pt_max); + auto myGen = new GeneratorPythia8LongLivedGapTriggered(mPdg, input_trigger_ratio, n_injected, pt_min, pt_max, y_min, y_max); myGen->setAlternatingPDGsign(alternate_sign); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); diff --git a/MC/run/PWGLF/run_NucleiFwdInjectedGap.sh b/MC/run/PWGLF/run_NucleiFwdInjectedGap.sh new file mode 100755 index 000000000..e83d1aed2 --- /dev/null +++ b/MC/run/PWGLF/run_NucleiFwdInjectedGap.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" +SIMENGINE=${SIMENGINE:-TGeant3} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-500000} +SYSTEM=${SYSTEM:-pp} +ENERGY=${ENERGY:-13600} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +# create workflow +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${ENERGY} -col ${SYSTEM} -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} -confKey "Diamond.width[0]=0.1;Diamond.width[1]=0.1;Diamond.width[2]=6." -e ${SIMENGINE} ${SEED} -mod "--skipModules ZDC" \ + -ini ${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFNucleiFwd${SYSTEM}Gap.ini + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 From d8b1e402af2d61600b657b6e34280d3df0c8f36f Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 26 Sep 2023 18:46:50 +0200 Subject: [PATCH 0097/1239] Change default option for strangeness tracking; Adjust threads for svfinder * change the default to no strangeness tracking (not validated and very slow) * adjust threads for secondary vertex finder to 8 since it can make use of this --- MC/bin/o2dpg_sim_workflow.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index e9d15a738..5967e218f 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -108,7 +108,7 @@ parser.add_argument('--no-combine-dpl-devices', action='store_true', help=argparse.SUPPRESS) parser.add_argument('--no-mc-labels', action='store_true', default=False, help=argparse.SUPPRESS) parser.add_argument('--no-tpc-digitchunking', action='store_true', help=argparse.SUPPRESS) -parser.add_argument('--no-strangeness-tracking', action='store_true', help=argparse.SUPPRESS) +parser.add_argument('--with-strangeness-tracking', action='store_true', default=False, help="Enable strangeness tracking") parser.add_argument('--combine-tpc-clusterization', action='store_true', help=argparse.SUPPRESS) #<--- useful for small productions (pp, low interaction rate, small number of events) parser.add_argument('--first-orbit', default=0, type=int, help=argparse.SUPPRESS) # to set the first orbit number of the run for HBFUtils (only used when anchoring) # (consider doing this rather in O2 digitization code directly) @@ -1275,8 +1275,8 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): svfinder_threads = ' --threads 1 ' svfinder_cpu = 1 if COLTYPE == "PbPb" or (doembedding and COLTYPEBKG == "PbPb"): - svfinder_threads = ' --threads 3 ' - svfinder_cpu = 3 + svfinder_threads = ' --threads 8 ' + svfinder_cpu = 8 SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000') SVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-secondary-vertexing-workflow ' SVFINDERtask['cmd'] += getDPL_global_options(bigshm=True) + svfinder_threads + putConfigValuesNew(['svertexer'], {"NameConf.mDirMatLUT" : ".."}) @@ -1288,7 +1288,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): svfinder_sources += ",MID" SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] # strangeness tracking is now called from the secondary vertexer - if args.no_strangeness_tracking: + if not args.with_strangeness_tracking: SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' # if enabled, it may require MC labels else: @@ -1356,7 +1356,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if environ.get('O2DPG_AOD_NOTRUNCATE') != None or environ.get('ALIEN_JDL_O2DPG_AOD_NOTRUNCATE') != None: AODtask['cmd'] += ' --enable-truncation 0' # developer option to suppress precision truncation - if args.no_strangeness_tracking: + if not args.with_strangeness_tracking: AODtask['cmd'] += ' --disable-strangeness-tracking' workflow['stages'].append(AODtask) From 8a545bf45eafea8964d3a71f46a6cb10309449e8 Mon Sep 17 00:00:00 2001 From: Luca Barioglio Date: Thu, 28 Sep 2023 12:15:57 +0200 Subject: [PATCH 0098/1239] Add generator for hypernuclei --- MC/config/PWGLF/ini/GeneratorLFHyperppGap.ini | 6 +++ .../PWGLF/ini/tests/GeneratorLFHyperppGap.C | 54 +++++++++++++++++++ MC/config/PWGLF/pythia8/generator/hyper.gun | 3 ++ 3 files changed, 63 insertions(+) create mode 100644 MC/config/PWGLF/ini/GeneratorLFHyperppGap.ini create mode 100644 MC/config/PWGLF/ini/tests/GeneratorLFHyperppGap.C create mode 100644 MC/config/PWGLF/pythia8/generator/hyper.gun diff --git a/MC/config/PWGLF/ini/GeneratorLFHyperppGap.ini b/MC/config/PWGLF/ini/GeneratorLFHyperppGap.ini new file mode 100644 index 000000000..e27b67891 --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFHyperppGap.ini @@ -0,0 +1,6 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/hyper.gun", 5) + +[GeneratorPythia8] +config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFHyperppGap.C b/MC/config/PWGLF/ini/tests/GeneratorLFHyperppGap.C new file mode 100644 index 000000000..27e56b2a7 --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFHyperppGap.C @@ -0,0 +1,54 @@ +int External() +{ + std::string path{"o2sim_Kine.root"}; + std::vector possiblePDGs = {1010010030, -1010010030, + 1010010040, -1010010040}; + + int nPossiblePDGs = possiblePDGs.size(); + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) + { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + if (!tree) + { + std::cerr << "Cannot find tree o2sim in file " << path << "\n"; + return 1; + } + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + std::vector injectedPDGs; + + auto nEvents = tree->GetEntries(); + for (int i = 0; i < nEvents; i++) + { + auto check = tree->GetEntry(i); + for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) + { + auto track = tracks->at(idxMCTrack); + auto pdg = track.GetPdgCode(); + auto it = std::find(possiblePDGs.begin(), possiblePDGs.end(), pdg); + if (it != possiblePDGs.end() && track.isPrimary()) // found + { + injectedPDGs.push_back(pdg); + } + } + } + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + if(injectedPDGs.empty()){ + std::cerr << "No injected particles\n"; + return 1; // At least one of the injected particles should be generated + } + for (int i = 0; i < nPossiblePDGs; i++) + { + std::cout << "# Injected nuclei \n"; + std::cout << possiblePDGs[i] << ": " << std::count(injectedPDGs.begin(), injectedPDGs.end(), possiblePDGs[i]) << "\n"; + } + return 0; +} diff --git a/MC/config/PWGLF/pythia8/generator/hyper.gun b/MC/config/PWGLF/pythia8/generator/hyper.gun new file mode 100644 index 000000000..881c7be3a --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/hyper.gun @@ -0,0 +1,3 @@ +# PDG N ptMin ptMax yMin yMax +1010010030 1 0.2 6 -1 1 +1010010040 1 0.2 6 -1 1 From 15fe7fc6d1c081182d6b994ec4e74d48cb15220e Mon Sep 17 00:00:00 2001 From: ffionda Date: Wed, 27 Sep 2023 13:02:47 +0200 Subject: [PATCH 0099/1239] include non-prompt psi(2S) and non-prompt Jpsi in the same production cycle --- .../BTOPSIJPSITODIELECTRON.DEC | 277 ++++++++++++++++++ .../DecayTablesEvtgen/BTOPSIJPSITODIMUON.DEC | 277 ++++++++++++++++++ MC/config/PWGDQ/EvtGen/GeneratorEvtGen.C | 8 + .../GeneratorBeautyToPsiAndJpsi_EvtGen.C | 84 ++++++ .../ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini | 22 ++ .../ini/GeneratorHF_bbbar_PsiAndJpsi_midy.ini | 28 ++ .../trigger/selectDaughterFromHFwithinAcc.C | 15 + 7 files changed, 711 insertions(+) create mode 100644 MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIELECTRON.DEC create mode 100644 MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIMUON.DEC create mode 100644 MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy.ini diff --git a/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIELECTRON.DEC b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIELECTRON.DEC new file mode 100644 index 000000000..64a835910 --- /dev/null +++ b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIELECTRON.DEC @@ -0,0 +1,277 @@ +### +Decay B0 +### +0.000310000 psi(2S) K_S0 SVS; #[Reconstructed PDG2011] +0.000310000 psi(2S) K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.000610000 psi(2S) K*0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0004 psi(2S) K+ pi- PHSP; +0.0002 psi(2S) K0 pi0 PHSP; +0.0002 psi(2S) K0 pi- pi+ PHSP; +0.0001 psi(2S) K0 pi0 pi0 PHSP; +0.0001 psi(2S) K+ pi- pi0 PHSP; +0.0004 psi(2S) K_10 PHSP; +### +0.000620000 psi(2S) K0 PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.000435500 J/psi K_S0 SVS; #[Reconstructed PDG2011] +0.000435500 J/psi K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.001330000 J/psi K*0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.000017600 J/psi pi0 SVS; #[Reconstructed PDG2011] +0.000027000 J/psi rho0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.00003 J/psi omega SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; +0.000000000 J/psi K+ pi- PHSP; #[Reconstructed PDG2011] +0.0001 J/psi K0 pi0 PHSP; +0.001300000 J/psi K_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.0001 J/psi K'_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*0 PHSP; +0.000094000 J/psi phi K0 PHSP; #[Reconstructed PDG2011] +#### +0.000871000 J/psi K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000310000 J/psi omega K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000009500 J/psi eta PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000019000 J/psi pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000460000 J/psi K0 pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000540000 J/psi K0 rho0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000800000 J/psi K*+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000660000 J/psi K*0 pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay anti-B0 +### +0.000310000 psi(2S) K_S0 SVS; #[Reconstructed PDG2011] +0.000310000 psi(2S) K_L0 SVS; #[Reconstructed PDG2011] +# +0.000610000 psi(2S) anti-K*0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0004 psi(2S) K- pi+ PHSP; +0.0002 psi(2S) anti-K0 pi0 PHSP; +0.0002 psi(2S) anti-K0 pi+ pi- PHSP; +0.0001 psi(2S) anti-K0 pi0 pi0 PHSP; +0.0001 psi(2S) K- pi+ pi0 PHSP; +0.0004 psi(2S) anti-K_10 PHSP; +### +0.000620000 psi(2S) anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.000435500 J/psi K_S0 SVS; #[Reconstructed PDG2011] +0.000435500 J/psi K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.001330000 J/psi anti-K*0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000017600 J/psi pi0 SVS; #[Reconstructed PDG2011] +0.000027000 J/psi rho0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000030 J/psi omega SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; +0.000000000 J/psi K- pi+ PHSP; #[Reconstructed PDG2011] +0.0001 J/psi anti-K0 pi0 PHSP; +0.001300000 J/psi anti-K_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.0001 J/psi anti-K'_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi anti-K_2*0 PHSP; +0.000094000 J/psi phi anti-K0 PHSP; #[Reconstructed PDG2011] +### +0.000871000 J/psi anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000310000 J/psi omega anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000009500 J/psi eta PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000019000 J/psi pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000460000 J/psi anti-K0 pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000540000 J/psi anti-K0 rho0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000800000 J/psi K*- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000660000 J/psi anti-K*0 pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B+ +### +0.000646000 psi(2S) K+ SVS; #[Reconstructed PDG2011] +0.000620000 psi(2S) K*+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0004 psi(2S) K0 pi+ PHSP; +0.0002 psi(2S) K+ pi0 PHSP; +0.001900000 psi(2S) K+ pi- pi+ PHSP; #[Reconstructed PDG2011] +0.0001 psi(2S) K+ pi0 pi0 PHSP; +0.0001 psi(2S) K0 pi+ pi0 PHSP; +0.0004 psi(2S) K_1+ PHSP; +### +0.000025800 psi(2S) pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.001014000 J/psi K+ SVS; #[Reconstructed PDG2011] +0.001430000 J/psi K*+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.000049000 J/psi pi+ SVS; #[Reconstructed PDG2011] +0.000050000 J/psi rho+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0002 J/psi K0 pi+ PHSP; +0.0001 J/psi K+ pi0 PHSP; +0.0001 J/psi K'_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*+ PHSP; +0.001800000 J/psi K_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.000052000 J/psi phi K+ PHSP; #[Reconstructed PDG2011] +# +#### +0.001070000 J/psi K+ pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000011800 J/psi p+ anti-Lambda0 PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B- +### +0.000646000 psi(2S) K- SVS; #[Reconstructed PDG2011] +0.000620000 psi(2S) K*- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0004 psi(2S) anti-K0 pi- PHSP; +0.0002 psi(2S) K- pi0 PHSP; +0.001900000 psi(2S) K- pi+ pi- PHSP; #[Reconstructed PDG2011] +0.0001 psi(2S) K- pi0 pi0 PHSP; +0.0001 psi(2S) anti-K0 pi- pi0 PHSP; +0.0004 psi(2S) K_1- PHSP; +### +0.000025800 psi(2S) pi- PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.001014000 J/psi K- SVS; #[Reconstructed PDG2011] +0.001430000 J/psi K*- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000049000 J/psi pi- SVS; #[Reconstructed PDG2011] +0.000050000 J/psi rho- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0002 J/psi anti-K0 pi- PHSP; +0.0001 J/psi K- pi0 PHSP; +0.0001 J/psi K'_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*- PHSP; +0.001800000 J/psi K_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.000052000 J/psi phi K- PHSP; #[Reconstructed PDG2011] +# +0.001070000 J/psi K- pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000011800 J/psi anti-p- Lambda0 PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B_s0 +### psi' = 0.34% CLNS 94/1315 +0.000465 psi(2S) eta' SVS; +0.000235 psi(2S) eta SVS; +0.000680000 psi(2S) phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.0003 psi(2S) K- K+ PHSP; +0.0003 psi(2S) anti-K0 K0 PHSP; +0.0003 psi(2S) K0 K- pi+ PHSP; +0.0003 psi(2S) anti-K0 K0 pi0 PHSP; +0.0003 psi(2S) K- K+ pi0 PHSP; +0.00034 psi(2S) phi pi+ pi- PHSP; +0.00034 psi(2S) phi pi0 pi0 PHSP; +0.0002 psi(2S) eta pi+ pi- PHSP; +0.0002 psi(2S) eta pi0 pi0 PHSP; +0.0004 psi(2S) eta' pi+ pi- PHSP; +0.0004 psi(2S) eta' pi0 pi0 PHSP; +0.0002 psi(2S) pi+ pi- PHSP; +0.0002 psi(2S) pi0 pi0 PHSP; +#### + +0.00064 J/psi eta' SVS; +0.00032 J/psi eta SVS; +0.001300000 J/psi phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.00008 J/psi K0 SVS; +0.00070 J/psi K- K+ PHSP; +0.00070 J/psi anti-K0 K0 PHSP; +0.00070 J/psi K0 K- pi+ PHSP; +0.00070 J/psi anti-K0 K0 pi0 PHSP; +0.00070 J/psi K- K+ pi0 PHSP; +# LHCb PR 04/02/04 Add (cc) phi n pi(+/0) +0.00039 J/psi phi pi+ pi- PHSP; +0.00039 J/psi phi pi0 pi0 PHSP; +# LHCb PR Add (cc) phi eta(') + npi like in CDF QQ +0.0002 J/psi eta pi+ pi- PHSP; +0.0002 J/psi eta pi0 pi0 PHSP; +0.0004 J/psi eta' pi+ pi- PHSP; +0.0004 J/psi eta' pi0 pi0 PHSP; +0.0002 J/psi pi+ pi- PHSP; +0.0002 J/psi pi0 pi0 PHSP; +# PR LHCb 04/08/2004 : add Bs -> phi mu mu, phi e e +0.0000023 phi e+ e- BTOSLLALI; +Enddecay + +Decay anti-B_s0 +### +0.000465 psi(2S) eta' SVS; +0.000235 psi(2S) eta SVS; +0.000680000 psi(2S) phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.0003 psi(2S) K- K+ PHSP; +0.0003 psi(2S) anti-K0 K0 PHSP; +0.0003 psi(2S) anti-K0 K+ pi- PHSP; +0.0003 psi(2S) anti-K0 K0 pi0 PHSP; +0.0003 psi(2S) K- K+ pi0 PHSP; +0.00034 psi(2S) phi pi+ pi- PHSP; +0.00034 psi(2S) phi pi0 pi0 PHSP; +0.0002 psi(2S) eta pi+ pi- PHSP; +0.0002 psi(2S) eta pi0 pi0 PHSP; +0.0004 psi(2S) eta' pi+ pi- PHSP; +0.0004 psi(2S) eta' pi0 pi0 PHSP; +0.0002 psi(2S) pi+ pi- PHSP; +0.0002 psi(2S) pi0 pi0 PHSP; +### +0.00064 J/psi eta' SVS; +0.00032 J/psi eta SVS; +0.001300000 J/psi phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.00008 J/psi K0 SVS; +0.00070 J/psi K- K+ PHSP; +0.00070 J/psi anti-K0 K0 PHSP; +0.00070 J/psi anti-K0 K+ pi- PHSP; +0.00070 J/psi anti-K0 K0 pi0 PHSP; +0.00070 J/psi K- K+ pi0 PHSP; +# LHCb PR 04/02/04 Add (cc) phi n pi(+/0) +0.00039 J/psi phi pi+ pi- PHSP; +0.00039 J/psi phi pi0 pi0 PHSP; +# LHCb PR add (cc) phi eta(') + npi see CDF QQ +0.0002 J/psi eta pi+ pi- PHSP; +0.0002 J/psi eta pi0 pi0 PHSP; +0.0004 J/psi eta' pi+ pi- PHSP; +0.0004 J/psi eta' pi0 pi0 PHSP; +0.0002 J/psi pi+ pi- PHSP; +0.0002 J/psi pi0 pi0 PHSP; +# PR LHCb 04/08/2004 : add Bs -> phi mu mu, phi e e +0.0000023 phi e- e+ BTOSLLALI; +Enddecay + +Decay Lambda_b0 +### +0.00038 Lambda0 psi(2S) PHSP; +0.00047 Lambda0 J/psi PHSP; +Enddecay + +Decay anti-Lambda_b0 +### +0.00038 anti-Lambda0 psi(2S) PHSP; +0.00047 anti-Lambda0 J/psi PHSP; +Enddecay + +Decay Xi_b- + 0.00047 Xi- J/psi PHSP; +Enddecay + + +Decay anti-Xi_b+ + 0.00047 anti-Xi+ J/psi PHSP; +Enddecay + +Decay Xi_b0 + 0.00047 Xi0 J/psi PHSP; +Enddecay + +Decay anti-Xi_b0 + 0.00047 anti-Xi0 J/psi PHSP; +Enddecay + +Decay Omega_b- + 0.00047 Omega- J/psi PHSP; +Enddecay + +Decay anti-Omega_b+ + 0.00047 anti-Omega+ J/psi PHSP; +Enddecay + + +Decay psi(2S) +### from DECAY.DEC +1.000 e+ e- PHOTOS VLL; +Enddecay + +Decay J/psi +1.000 e+ e- PHOTOS VLL; +Enddecay + +End + diff --git a/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIMUON.DEC b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIMUON.DEC new file mode 100644 index 000000000..dbcfc5bd8 --- /dev/null +++ b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BTOPSIJPSITODIMUON.DEC @@ -0,0 +1,277 @@ +### +Decay B0 +### +0.000310000 psi(2S) K_S0 SVS; #[Reconstructed PDG2011] +0.000310000 psi(2S) K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.000610000 psi(2S) K*0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0004 psi(2S) K+ pi- PHSP; +0.0002 psi(2S) K0 pi0 PHSP; +0.0002 psi(2S) K0 pi- pi+ PHSP; +0.0001 psi(2S) K0 pi0 pi0 PHSP; +0.0001 psi(2S) K+ pi- pi0 PHSP; +0.0004 psi(2S) K_10 PHSP; +### +0.000620000 psi(2S) K0 PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.000435500 J/psi K_S0 SVS; #[Reconstructed PDG2011] +0.000435500 J/psi K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.001330000 J/psi K*0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.000017600 J/psi pi0 SVS; #[Reconstructed PDG2011] +0.000027000 J/psi rho0 SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.00003 J/psi omega SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; +0.000000000 J/psi K+ pi- PHSP; #[Reconstructed PDG2011] +0.0001 J/psi K0 pi0 PHSP; +0.001300000 J/psi K_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.0001 J/psi K'_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*0 PHSP; +0.000094000 J/psi phi K0 PHSP; #[Reconstructed PDG2011] +#### +0.000871000 J/psi K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000310000 J/psi omega K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000009500 J/psi eta PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000019000 J/psi pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000460000 J/psi K0 pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000540000 J/psi K0 rho0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000800000 J/psi K*+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000660000 J/psi K*0 pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay anti-B0 +### +0.000310000 psi(2S) K_S0 SVS; #[Reconstructed PDG2011] +0.000310000 psi(2S) K_L0 SVS; #[Reconstructed PDG2011] +# +0.000610000 psi(2S) anti-K*0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0004 psi(2S) K- pi+ PHSP; +0.0002 psi(2S) anti-K0 pi0 PHSP; +0.0002 psi(2S) anti-K0 pi+ pi- PHSP; +0.0001 psi(2S) anti-K0 pi0 pi0 PHSP; +0.0001 psi(2S) K- pi+ pi0 PHSP; +0.0004 psi(2S) anti-K_10 PHSP; +### +0.000620000 psi(2S) anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.000435500 J/psi K_S0 SVS; #[Reconstructed PDG2011] +0.000435500 J/psi K_L0 SVS; #[Reconstructed PDG2011] +# +# +0.001330000 J/psi anti-K*0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000017600 J/psi pi0 SVS; #[Reconstructed PDG2011] +0.000027000 J/psi rho0 SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000030 J/psi omega SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; +0.000000000 J/psi K- pi+ PHSP; #[Reconstructed PDG2011] +0.0001 J/psi anti-K0 pi0 PHSP; +0.001300000 J/psi anti-K_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.0001 J/psi anti-K'_10 SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi anti-K_2*0 PHSP; +0.000094000 J/psi phi anti-K0 PHSP; #[Reconstructed PDG2011] +### +0.000871000 J/psi anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000310000 J/psi omega anti-K0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000009500 J/psi eta PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000019000 J/psi pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000460000 J/psi anti-K0 pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000540000 J/psi anti-K0 rho0 PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000800000 J/psi K*- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000660000 J/psi anti-K*0 pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B+ +### +0.000646000 psi(2S) K+ SVS; #[Reconstructed PDG2011] +0.000620000 psi(2S) K*+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0004 psi(2S) K0 pi+ PHSP; +0.0002 psi(2S) K+ pi0 PHSP; +0.001900000 psi(2S) K+ pi- pi+ PHSP; #[Reconstructed PDG2011] +0.0001 psi(2S) K+ pi0 pi0 PHSP; +0.0001 psi(2S) K0 pi+ pi0 PHSP; +0.0004 psi(2S) K_1+ PHSP; +### +0.000025800 psi(2S) pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.001014000 J/psi K+ SVS; #[Reconstructed PDG2011] +0.001430000 J/psi K*+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.000049000 J/psi pi+ SVS; #[Reconstructed PDG2011] +0.000050000 J/psi rho+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0002 J/psi K0 pi+ PHSP; +0.0001 J/psi K+ pi0 PHSP; +0.0001 J/psi K'_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*+ PHSP; +0.001800000 J/psi K_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.000052000 J/psi phi K+ PHSP; #[Reconstructed PDG2011] +# +#### +0.001070000 J/psi K+ pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000011800 J/psi p+ anti-Lambda0 PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B- +### +0.000646000 psi(2S) K- SVS; #[Reconstructed PDG2011] +0.000620000 psi(2S) K*- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0004 psi(2S) anti-K0 pi- PHSP; +0.0002 psi(2S) K- pi0 PHSP; +0.001900000 psi(2S) K- pi+ pi- PHSP; #[Reconstructed PDG2011] +0.0001 psi(2S) K- pi0 pi0 PHSP; +0.0001 psi(2S) anti-K0 pi- pi0 PHSP; +0.0004 psi(2S) K_1- PHSP; +### +0.000025800 psi(2S) pi- PHSP; #[New mode added] #[Reconstructed PDG2011] + +0.001014000 J/psi K- SVS; #[Reconstructed PDG2011] +0.001430000 J/psi K*- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.000049000 J/psi pi- SVS; #[Reconstructed PDG2011] +0.000050000 J/psi rho- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0002 J/psi anti-K0 pi- PHSP; +0.0001 J/psi K- pi0 PHSP; +0.0001 J/psi K'_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +0.0005 J/psi K_2*- PHSP; +0.001800000 J/psi K_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] +0.000052000 J/psi phi K- PHSP; #[Reconstructed PDG2011] +# +0.001070000 J/psi K- pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000011800 J/psi anti-p- Lambda0 PHSP; #[New mode added] #[Reconstructed PDG2011] +Enddecay + +Decay B_s0 +### psi' = 0.34% CLNS 94/1315 +0.000465 psi(2S) eta' SVS; +0.000235 psi(2S) eta SVS; +0.000680000 psi(2S) phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.0003 psi(2S) K- K+ PHSP; +0.0003 psi(2S) anti-K0 K0 PHSP; +0.0003 psi(2S) K0 K- pi+ PHSP; +0.0003 psi(2S) anti-K0 K0 pi0 PHSP; +0.0003 psi(2S) K- K+ pi0 PHSP; +0.00034 psi(2S) phi pi+ pi- PHSP; +0.00034 psi(2S) phi pi0 pi0 PHSP; +0.0002 psi(2S) eta pi+ pi- PHSP; +0.0002 psi(2S) eta pi0 pi0 PHSP; +0.0004 psi(2S) eta' pi+ pi- PHSP; +0.0004 psi(2S) eta' pi0 pi0 PHSP; +0.0002 psi(2S) pi+ pi- PHSP; +0.0002 psi(2S) pi0 pi0 PHSP; +#### + +0.00064 J/psi eta' SVS; +0.00032 J/psi eta SVS; +0.001300000 J/psi phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.00008 J/psi K0 SVS; +0.00070 J/psi K- K+ PHSP; +0.00070 J/psi anti-K0 K0 PHSP; +0.00070 J/psi K0 K- pi+ PHSP; +0.00070 J/psi anti-K0 K0 pi0 PHSP; +0.00070 J/psi K- K+ pi0 PHSP; +# LHCb PR 04/02/04 Add (cc) phi n pi(+/0) +0.00039 J/psi phi pi+ pi- PHSP; +0.00039 J/psi phi pi0 pi0 PHSP; +# LHCb PR Add (cc) phi eta(') + npi like in CDF QQ +0.0002 J/psi eta pi+ pi- PHSP; +0.0002 J/psi eta pi0 pi0 PHSP; +0.0004 J/psi eta' pi+ pi- PHSP; +0.0004 J/psi eta' pi0 pi0 PHSP; +0.0002 J/psi pi+ pi- PHSP; +0.0002 J/psi pi0 pi0 PHSP; +# PR LHCb 04/08/2004 : add Bs -> phi mu mu, phi e e +0.0000023 phi e+ e- BTOSLLALI; +Enddecay + +Decay anti-B_s0 +### +0.000465 psi(2S) eta' SVS; +0.000235 psi(2S) eta SVS; +0.000680000 psi(2S) phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.0003 psi(2S) K- K+ PHSP; +0.0003 psi(2S) anti-K0 K0 PHSP; +0.0003 psi(2S) anti-K0 K+ pi- PHSP; +0.0003 psi(2S) anti-K0 K0 pi0 PHSP; +0.0003 psi(2S) K- K+ pi0 PHSP; +0.00034 psi(2S) phi pi+ pi- PHSP; +0.00034 psi(2S) phi pi0 pi0 PHSP; +0.0002 psi(2S) eta pi+ pi- PHSP; +0.0002 psi(2S) eta pi0 pi0 PHSP; +0.0004 psi(2S) eta' pi+ pi- PHSP; +0.0004 psi(2S) eta' pi0 pi0 PHSP; +0.0002 psi(2S) pi+ pi- PHSP; +0.0002 psi(2S) pi0 pi0 PHSP; +### +0.00064 J/psi eta' SVS; +0.00032 J/psi eta SVS; +0.001300000 J/psi phi SVV_HELAMP 1.0 0.0 1.0 0.0 1.0 0.0; #[Reconstructed PDG2011] +0.00008 J/psi K0 SVS; +0.00070 J/psi K- K+ PHSP; +0.00070 J/psi anti-K0 K0 PHSP; +0.00070 J/psi anti-K0 K+ pi- PHSP; +0.00070 J/psi anti-K0 K0 pi0 PHSP; +0.00070 J/psi K- K+ pi0 PHSP; +# LHCb PR 04/02/04 Add (cc) phi n pi(+/0) +0.00039 J/psi phi pi+ pi- PHSP; +0.00039 J/psi phi pi0 pi0 PHSP; +# LHCb PR add (cc) phi eta(') + npi see CDF QQ +0.0002 J/psi eta pi+ pi- PHSP; +0.0002 J/psi eta pi0 pi0 PHSP; +0.0004 J/psi eta' pi+ pi- PHSP; +0.0004 J/psi eta' pi0 pi0 PHSP; +0.0002 J/psi pi+ pi- PHSP; +0.0002 J/psi pi0 pi0 PHSP; +# PR LHCb 04/08/2004 : add Bs -> phi mu mu, phi e e +0.0000023 phi e- e+ BTOSLLALI; +Enddecay + +Decay Lambda_b0 +### +0.00038 Lambda0 psi(2S) PHSP; +0.00047 Lambda0 J/psi PHSP; +Enddecay + +Decay anti-Lambda_b0 +### +0.00038 anti-Lambda0 psi(2S) PHSP; +0.00047 anti-Lambda0 J/psi PHSP; +Enddecay + +Decay Xi_b- + 0.00047 Xi- J/psi PHSP; +Enddecay + + +Decay anti-Xi_b+ + 0.00047 anti-Xi+ J/psi PHSP; +Enddecay + +Decay Xi_b0 + 0.00047 Xi0 J/psi PHSP; +Enddecay + +Decay anti-Xi_b0 + 0.00047 anti-Xi0 J/psi PHSP; +Enddecay + +Decay Omega_b- + 0.00047 Omega- J/psi PHSP; +Enddecay + +Decay anti-Omega_b+ + 0.00047 anti-Omega+ J/psi PHSP; +Enddecay + + +Decay psi(2S) +### from DECAY.DEC +1.000 mu+ mu- PHOTOS VLL; +Enddecay + +Decay J/psi +1.000 mu+ mu- PHOTOS VLL; +Enddecay + +End + diff --git a/MC/config/PWGDQ/EvtGen/GeneratorEvtGen.C b/MC/config/PWGDQ/EvtGen/GeneratorEvtGen.C index df15cd947..7c0f4650e 100644 --- a/MC/config/PWGDQ/EvtGen/GeneratorEvtGen.C +++ b/MC/config/PWGDQ/EvtGen/GeneratorEvtGen.C @@ -19,6 +19,8 @@ enum DecayModeEvt { kEvtAll = 0, kEvtBJpsiDiMuon, kEvtBPsiDiElectron, kEvtBPsiDiMuon, + kEvtBPsiAndJpsiDiElectron, + kEvtBPsiAndJpsiDiMuon, kEvtBSemiElectronic, kEvtHadronicD, kEvtHadronicDWithout4Bodies, @@ -281,6 +283,12 @@ class GeneratorEvtGen : public T case kEvtBPsiDiMuon: SetDecayTable(Form("%s/BTOPSITOMU.DEC", pathO2.Data())); break; + case kEvtBPsiAndJpsiDiElectron: + SetDecayTable(Form("%s/BTOPSIJPSITODIELECTRON.DEC", pathO2.Data())); + break; + case kEvtBPsiAndJpsiDiMuon: + SetDecayTable(Form("%s/BTOPSIJPSITODIMUON.DEC", pathO2.Data())); + break; case kEvtBSemiElectronic: SetDecayTable(Form("%s/BTOELE.DEC", pathO2.Data())); break; diff --git a/MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C b/MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C new file mode 100644 index 000000000..98a4763c6 --- /dev/null +++ b/MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C @@ -0,0 +1,84 @@ +// usage (fwdy) : +// o2-sim -j 4 -n 10 -g external -t external -m "PIPE ITS TPC" -o sgn --configFile GeneratorHF_bbbar_PsiAndJpsi_midy.ini +// usage (midy) : +// o2-sim -j 4 -n 10 -g external -t external -m "PIPE ITS TPC" -o sgn --configFile GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini +// +// +R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGHF/external/generator) +#include "GeneratorEvtGen.C" +#include "GeneratorHF.C" + +FairGenerator* + GeneratorBeautyToPsiAndJpsi_EvtGenMidY(double rapidityMin = -1.5, double rapidityMax = 1.5, bool ispp = true, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + + gen->setVerbose(verbose); + if (ispp) + gen->setFormula("1"); + else + gen->setFormula("max(1.,120.*(x<5.)+80.*(1.-x/20.)*(x>5.)*(x<11.)+240.*(1.-x/13.)*(x>11.))"); + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBPsiAndJpsiDiElectron); + + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + // print debug + // gen->PrintDebug(); + + return gen; +} + +FairGenerator* + GeneratorBeautyToPsiAndJpsi_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = -2.2, bool ispp = true, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + + gen->setVerbose(verbose); + if (ispp) + gen->setFormula("1"); + else + gen->setFormula("max(1.,120.*(x<5.)+80.*(1.-x/20.)*(x>5.)*(x<11.)+240.*(1.-x/13.)*(x>11.))"); + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBPsiAndJpsiDiMuon); + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + // print debug + // gen->PrintDebug(); + + return gen; +} diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini new file mode 100644 index 000000000..9833b8e6e --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini @@ -0,0 +1,22 @@ +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C +funcName = GeneratorBeautyToPsiAndJpsi_EvtGenFwdY() + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-4.3,-2.3) + +### The setup uses an external even generator trigger which is +### defined in the following file and it is retrieved and configured +### according to the specified function call + +[TriggerExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C +funcName = selectDaughterFromHFwithinAcc("443;100443",kTRUE,-4.3,-2.3) diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy.ini new file mode 100644 index 000000000..0a6d69f54 --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy.ini @@ -0,0 +1,28 @@ +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorBeautyToPsiAndJpsi_EvtGen.C +funcName = GeneratorBeautyToPsiAndJpsi_EvtGenMidY() + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-1.5,1.5) + +### The setup uses an external even generator trigger which is +### defined in the following file and it is retrieved and configured +### according to the specified function call + +[TriggerExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C +funcName = selecMultipletHFwithinAcc("443;100443",kTRUE,-1.5,1.5) + +### The setup inhibits transport of primary particles which are produce at forward rapidity. +### The settings below only transports particles in the barrel, which is currently defined as |eta| < 2 + +[Stack] +transportPrimary = barrel diff --git a/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C b/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C index e37f8325b..44035a834 100644 --- a/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C +++ b/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C @@ -90,6 +90,21 @@ o2::eventgen::Trigger selectHFwithinAcc(Int_t pdgPartForAccCut = 521, Bool_t cut }; } +o2::eventgen::Trigger selecMultipletHFwithinAcc(TString pdgPartForAccCut="443;100443", Bool_t cutonSinglePart = kTRUE, double rapidityMin = -1., double rapidityMax = -1., int minNb = -1) +{ + return [pdgPartForAccCut, cutonSinglePart, rapidityMin, rapidityMax, minNb](const std::vector& particles) -> bool { + TObjArray* obj = pdgPartForAccCut.Tokenize(";"); + std::string spdg; Int_t pdgCode = -1; + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + pdgCode = std::stoi(spdg); + if(selectDaughterFromHFwithinAcc(pdgCode,cutonSinglePart,rapidityMin,rapidityMax,minNb)) { return kTRUE; } + + } + return kFALSE; + }; +} + Int_t GetFlavour(Int_t pdgCode) { // From 5918f86775c224139eeee82b2678081a2ddc38f8 Mon Sep 17 00:00:00 2001 From: ffionda Date: Thu, 28 Sep 2023 09:20:24 +0200 Subject: [PATCH 0100/1239] fix generator test macros --- .../ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini | 2 +- .../tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy.C | 92 +++++++++++++++++++ .../tests/GeneratorHF_bbbar_PsiAndJpsi_midy.C | 92 +++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy.C create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy.C diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini index 9833b8e6e..b4c6801e8 100644 --- a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy.ini @@ -19,4 +19,4 @@ hooksFuncName = pythia8_userhooks_bbbar(-4.3,-2.3) [TriggerExternal] fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C -funcName = selectDaughterFromHFwithinAcc("443;100443",kTRUE,-4.3,-2.3) +funcName = selecMultipletHFwithinAcc("443;100443",kTRUE,-4.3,-2.3) diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy.C new file mode 100644 index 000000000..52d699151 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy.C @@ -0,0 +1,92 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 13; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Int_t bpdgs[] = {511, 521, 531, 5112, 5122, 5232, 5132}; + Int_t sizePdg = sizeof(bpdgs)/sizeof(Int_t); + Bool_t hasBeautyMoth = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + // check if mothers are beauty hadrons + hasBeautyMoth = kFALSE; + if(idMoth){ // check beauty mother + auto tdM = mcreader.getTrack(i, idMoth); + for(int i=0; iGetPdgCode()) == bpdgs[i] ) hasBeautyMoth = kTRUE; } + } + if(hasBeautyMoth){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(rapidity > -4.3 && rapidity < -2.3) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (jpsi <- b): " << nSignalJpsi << "; within acceptance (-4.3 < y < -2.3): " << nSignalJpsiWithinAcc << "\n" + << "#signal (psi2S <- b): " << nSignalPsi2S << "; within acceptance (-4.3 < y < -2.3): " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy.C new file mode 100644 index 000000000..8a0964910 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy.C @@ -0,0 +1,92 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 11; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Int_t bpdgs[] = {511, 521, 531, 5112, 5122, 5232, 5132}; + Int_t sizePdg = sizeof(bpdgs)/sizeof(Int_t); + Bool_t hasBeautyMoth = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + // check if mothers are beauty hadrons + hasBeautyMoth = kFALSE; + if(idMoth){ // check beauty mother + auto tdM = mcreader.getTrack(i, idMoth); + for(int i=0; iGetPdgCode()) == bpdgs[i] ) hasBeautyMoth = kTRUE; } + } + if(hasBeautyMoth){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(std::abs(rapidity) < 1.0) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (jpsi <- b): " << nSignalJpsi << "; within acceptance (|y| < 1): " << nSignalJpsiWithinAcc << "\n" + << "#signal (psi2S <- b): " << nSignalPsi2S << "; within acceptance (|y| < 1): " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} From ce73557469ab58a714090e2b652af883d4cd2f37 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Thu, 28 Sep 2023 23:19:30 +0200 Subject: [PATCH 0101/1239] Scaling of TPC corr maps for PbPb 2023 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index ad5ced062..d06574edf 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -313,6 +313,11 @@ elif [[ $ALIGNLEVEL == 1 ]]; then export TPC_CORR_SCALING+=" --corrmap-lumi-mean $ALIEN_JDL_MEANIRFORTPC " fi + if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]]; then + unset TPC_CORR_SCALING + export TPC_CORR_SCALING="--ctp-lumi-factor 2.414 --require-ctp-lumi" + fi + if [[ $PERIOD != @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]] ; then echo "Setting TPCCLUSTERTIMESHIFT to 0" TPCCLUSTERTIMESHIFT=0 From 707fb5deb537d64b91569493fd756425ab39442d Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Fri, 29 Sep 2023 08:56:32 +0200 Subject: [PATCH 0102/1239] Change path to qc files for calibration files (#1249) --- DATA/production/calib/tpc-laser-filter.sh | 2 +- DATA/production/calib/tpc-pedestal.sh | 2 +- DATA/production/calib/tpc-pulser.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index e037197b9..3f26160e9 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -48,7 +48,7 @@ LASER_DECODER_ADD='' HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-laser-calib-qcmn" if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" diff --git a/DATA/production/calib/tpc-pedestal.sh b/DATA/production/calib/tpc-pedestal.sh index b60bcf28c..31039db4f 100755 --- a/DATA/production/calib/tpc-pedestal.sh +++ b/DATA/production/calib/tpc-pedestal.sh @@ -19,7 +19,7 @@ CCDB_PATH="http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn" max_events=50 publish_after=400 diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index eb45e3a6a..9541e42ca 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -20,7 +20,7 @@ CCDB_PATH="http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn" max_events=200 publish_after=230 From eba174242a6a04b3c4aa51b95c97b17be9d0c49a Mon Sep 17 00:00:00 2001 From: noferini Date: Fri, 29 Sep 2023 10:10:33 +0200 Subject: [PATCH 0103/1239] remove ft0 usage from TOF calib --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index d06574edf..94684afc4 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -405,7 +405,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching From bae468e4754e57d5f1a16ada485214a366345769 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Fri, 29 Sep 2023 09:25:56 +0200 Subject: [PATCH 0104/1239] Align TPC reco command --- MC/bin/o2dpg_sim_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 5967e218f..4d1afb45c 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -928,8 +928,9 @@ def getDigiTaskName(det): tpcclustertasks.append(taskname) tpcclussect = createTask(name=taskname, needs=[TPCDigitask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='2', mem='8000') digitmergerstr = '${O2_ROOT}/bin/o2-tpc-chunkeddigit-merger --tpc-sectors ' + str(s)+'-'+str(s+sectorpertask-1) + ' --tpc-lanes ' + str(NWORKERS) + ' | ' - tpcclussect['cmd'] = (digitmergerstr,'')[args.no_tpc_digitchunking] + ' ${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options(bigshm=True) + ' --input-type ' + ('digitizer','digits')[args.no_tpc_digitchunking] + ' --output-type clusters,send-clusters-per-sector --outfile tpc-native-clusters-part' + str((int)(s/sectorpertask)) + '.root --tpc-sectors ' + str(s)+'-'+str(s+sectorpertask-1) + ' ' + putConfigValuesNew(["GPU_global"], {"GPU_proc.ompThreads" : 4}) + ('',' --disable-mc')[args.no_mc_labels] + tpcclussect['cmd'] = (digitmergerstr,'')[args.no_tpc_digitchunking] + ' ${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options(bigshm=True) + ' --input-type ' + ('digitizer','digits')[args.no_tpc_digitchunking] + ' --output-type clusters,send-clusters-per-sector --tpc-native-cluster-writer \" --outfile tpc-native-clusters-part'+ str((int)(s/sectorpertask)) + '.root\" --tpc-sectors ' + str(s)+'-'+str(s+sectorpertask-1) + ' ' + putConfigValuesNew(["GPU_global"], {"GPU_proc.ompThreads" : 4}) + ('',' --disable-mc')[args.no_mc_labels] tpcclussect['env'] = { "OMP_NUM_THREADS" : "4", "SHMSIZE" : "16000000000" } + tpcclussect['semaphore'] = "tpctriggers.root" tpcclussect['retry_count'] = 2 # the task has a race condition --> makes sense to retry workflow['stages'].append(tpcclussect) From 7df264297716cb3878e9e2117c9d5c64048b1b4f Mon Sep 17 00:00:00 2001 From: noferini Date: Fri, 29 Sep 2023 15:29:15 +0200 Subject: [PATCH 0105/1239] re-enable fit in TOF calibs --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 94684afc4..d06574edf 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -405,7 +405,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching From 692369876326c32c4b57eeb8dd53a179a447a2b4 Mon Sep 17 00:00:00 2001 From: Luca Barioglio Date: Fri, 29 Sep 2023 15:35:51 +0200 Subject: [PATCH 0106/1239] Add run task --- MC/run/PWGLF/run_HyperInjectedGap.sh | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 MC/run/PWGLF/run_HyperInjectedGap.sh diff --git a/MC/run/PWGLF/run_HyperInjectedGap.sh b/MC/run/PWGLF/run_HyperInjectedGap.sh new file mode 100644 index 000000000..a973ac443 --- /dev/null +++ b/MC/run/PWGLF/run_HyperInjectedGap.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" +SIMENGINE=${SIMENGINE:-TGeant4} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-500000} +SYSTEM=${SYSTEM:-pp} +ENERGY=${ENERGY:-13600} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +# create workflow +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${ENERGY} -col ${SYSTEM} -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} -confKey "Diamond.width[0]=0.1;Diamond.width[1]=0.1;Diamond.width[2]=6." -e ${SIMENGINE} ${SEED} -mod "--skipModules ZDC" \ + -ini ${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFHyper${SYSTEM}Gap.ini + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 From 1a1cf15d98b201f6337f1eec3c36f22fbfa3904b Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 3 Oct 2023 16:40:25 +0200 Subject: [PATCH 0107/1239] Updating GLO json, following PR O2-11974, QC-2001 --- DATA/production/qc-async/itstpc.json | 2 +- DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/itstpc.json b/DATA/production/qc-async/itstpc.json index 9854b426a..9abcd46c7 100644 --- a/DATA/production/qc-async/itstpc.json +++ b/DATA/production/qc-async/itstpc.json @@ -39,7 +39,7 @@ "dataSource" : { "type" : "direct", "query_comment" : "checking every matched track", - "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS" + "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackITS:ITS/TRACKS/0;trackITSROF:ITS/ITSTrackROF/0;trackITSClIdx:ITS/TRACKCLSID/0;alpparITS:ITS/ALPIDEPARAM/0?lifetime=condition&ccdb-path=ITS/Config/AlpideParam" }, "taskParameters" : { "GID" : "ITS-TPC,ITS", diff --git a/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json b/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json index 38edce09e..da8964a41 100644 --- a/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json +++ b/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json @@ -70,7 +70,7 @@ "active" : "true", "machines" : [], "query_comment" : "checking every 10% matched track", - "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS", + "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackITS:ITS/TRACKS/0;trackITSROF:ITS/ITSTrackROF/0;trackITSClIdx:ITS/TRACKCLSID/0;alpparITS:ITS/ALPIDEPARAM/0?lifetime=condition&ccdb-path=ITS/Config/AlpideParam", "samplingConditions" : [ { "condition" : "random", From 38f90158928f1827131c2102af2b9fe8e12fb34a Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Fri, 29 Sep 2023 16:32:18 +0200 Subject: [PATCH 0108/1239] [AnalysisQC] Account for collision system * specific config files, new structure |--default |--pp |--analysis-testing-data.json |--analysis-testing-mc.json |--pbpb |--analysis-testing-data.json |--analysis-testing-mc.json |-- |--pp |--analysis-testing-data.json (optional) |--analysis-testing-mc.json (optional) |--pbpb |--analysis-testing-data.json (optional) |--analysis-testing-mc.json (optional) config files for specifig analysis are all optional. Whenever something cannot be found, it will fallback to the default * derive collision system in descending precendence 1. take what the user passed with --collision-system 2. try to derived from ALIEN_JDL_LPMInteractionType 3. fallback to pp * apply necessary changes to o2dpg_sim_workflow.py --- .../o2dpg_analysis_test_workflow.py | 66 +- MC/bin/o2dpg_sim_workflow.py | 2 +- .../pbpb/analysis-testing-data.json | 568 +++++++ .../pbpb/analysis-testing-data.json} | 0 .../pp/analysis-testing-data.json | 106 ++ .../pbpb/analysis-testing-data.json} | 0 .../TPCSkimming/pbpb/analysis-testing-mc.json | 315 ++++ .../TPCSkimming/pp/analysis-testing-data.json | 315 ++++ .../TPCSkimming/pp/analysis-testing-mc.json | 315 ++++ .../json/analyses_config.json | 96 +- .../pbpb}/analysis-testing-data.json | 0 .../pbpb}/analysis-testing-mc.json | 0 .../default/pp/analysis-testing-data.json | 567 +++++++ .../json/default/pp/analysis-testing-mc.json | 1442 +++++++++++++++++ 14 files changed, 3720 insertions(+), 72 deletions(-) create mode 100644 MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json rename MC/config/analysis_testing/json/{analysis-testing-EventTrackQA-data.json => EventTrackQA/pbpb/analysis-testing-data.json} (100%) create mode 100644 MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json rename MC/config/analysis_testing/json/{analysis-testing-TPCSkimming.json => TPCSkimming/pbpb/analysis-testing-data.json} (100%) create mode 100644 MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-mc.json create mode 100644 MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-data.json create mode 100644 MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-mc.json rename MC/config/analysis_testing/json/{ => default/pbpb}/analysis-testing-data.json (100%) rename MC/config/analysis_testing/json/{ => default/pbpb}/analysis-testing-mc.json (100%) create mode 100644 MC/config/analysis_testing/json/default/pp/analysis-testing-data.json create mode 100644 MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 45b32bacd..2d6f11cf8 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -91,6 +91,48 @@ ANALYSIS_LABEL_ON_MC = f"{ANALYSIS_LABEL}MC" ANALYSIS_VALID_MC = "mc" ANALYSIS_VALID_DATA = "data" +ANALYSIS_COLLISION_SYSTEM_PP = "pp" +ANALYSIS_COLLISION_SYSTEM_PBPB = "pbpb" +ANALYSIS_CONFIGURATION_PREFIX = "analysis-testing" +ANALYSIS_DEFAULT_CONFIGURATION = {ANALYSIS_COLLISION_SYSTEM_PP: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), + ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}, + ANALYSIS_COLLISION_SYSTEM_PBPB: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), + ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}} + + +def sanitize_configuration_path(path): + # sanitize path + path = path.replace("json://", "") + if path[0] != "$": + # only do this if there is no potential environment variable given as the first part of the path + path = abspath(expanduser(path)) + return f"json://{path}" + + +def get_default_configuration(data_or_mc, collision_system): + path = ANALYSIS_DEFAULT_CONFIGURATION.get(collision_system, None) + if not path: + print(f"ERROR: Unknown collision system {collision_system}") + return None + return path[data_or_mc] + + +def get_configuration(analysis_name, data_or_mc, collision_system): + path = join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", analysis_name, collision_system, f"{ANALYSIS_CONFIGURATION_PREFIX}-{data_or_mc}.json") + if not exists(path): + path = get_default_configuration(data_or_mc, collision_system) + if not path: + return None + print(f"INFO: Use default configuration for {analysis_name}") + return sanitize_configuration_path(path) + + return sanitize_configuration_path(path) + + +def get_collision_system(collision_system=None): + if not collision_system: + return environ.get("ALIEN_JDL_LPMINTERACTIONTYPE", "pp").lower() + return collision_system.lower() def full_ana_name(raw_ana_name): @@ -228,7 +270,7 @@ def get_additional_workflows(input_aod): return additional_workflows -def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis", *, analyses_only=None, is_mc=True, needs=None, autoset_converters=False, include_disabled_analyses=False, timeout=None): +def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis", *, analyses_only=None, is_mc=True, collision_system=None, needs=None, autoset_converters=False, include_disabled_analyses=False, timeout=None): """Add default analyses to user workflow Args: @@ -257,18 +299,21 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis additional_workflows = get_additional_workflows(input_aod) data_or_mc = ANALYSIS_VALID_MC if is_mc else ANALYSIS_VALID_DATA + collision_system = get_collision_system(collision_system) for ana in load_analyses(analyses_only, include_disabled_analyses=include_disabled_analyses): - configuration = ana.get("config", {}).get(data_or_mc, None) + if is_mc and not ana.get("valid_mc", False): + print(f"INFO: Analysis {ana['name']} not added since not valid in MC") + continue + if not is_mc and not ana.get("valid_data", False): + print(f"INFO: Analysis {ana['name']} not added since not valid in data") + continue + + configuration = get_configuration(ana["name"], data_or_mc, collision_system) if not configuration: - print(f"INFO: Analysis {ana['name']} not added since no configuration found for {data_or_mc}") + print(f"INFO: Analysis {ana['name']} excluded due to no valid configuration") continue - # sanitize path - configuration = configuration.replace("json://", "") - if configuration[0] != "$": - # only do this if there is no potential environment variable given as the first part of the path - configuration = abspath(expanduser(configuration)) - configuration = f"json://{configuration}" + print(f"INFO: Analysis {ana['name']} uses configuration {configuration}") for i in additional_workflows: if i not in ana["tasks"]: @@ -335,7 +380,7 @@ def run(args): return 1 workflow = [] - add_analysis_tasks(workflow, args.input_file, expanduser(args.analysis_dir), is_mc=args.is_mc, analyses_only=args.only_analyses, autoset_converters=args.autoset_converters, include_disabled_analyses=args.include_disabled, timeout=args.timeout) + add_analysis_tasks(workflow, args.input_file, expanduser(args.analysis_dir), is_mc=args.is_mc, analyses_only=args.only_analyses, autoset_converters=args.autoset_converters, include_disabled_analyses=args.include_disabled, timeout=args.timeout, collision_system=args.collision_system) if args.with_qc_upload: add_analysis_qc_upload_tasks(workflow, args.period_name, args.run_number, args.pass_name) if not workflow: @@ -360,6 +405,7 @@ def main(): parser.add_argument("--include-disabled", dest="include_disabled", action="store_true", help="ignore if an analysis is disabled an run anyway") parser.add_argument("--autoset-converters", dest="autoset_converters", action="store_true", help="Compatibility mode to automatically set the converters for the analysis") parser.add_argument("--timeout", type=int, default=None, help="Timeout for analysis tasks in seconds.") + parser.add_argument("--collision-system", dest="collision_system", help="Set the collision system. If not set, tried to be derived from ALIEN_JDL_LPMInterationType. Fallback to pp") parser.set_defaults(func=run) args = parser.parse_args() return(args.func(args)) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 4d1afb45c..3753c4ad9 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1410,7 +1410,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if includeAnalysis: # include analyses and potentially final QC upload tasks - add_analysis_tasks(workflow["stages"], needs=[AOD_merge_task["name"]], is_mc=True) + add_analysis_tasks(workflow["stages"], needs=[AOD_merge_task["name"]], is_mc=True, collision_system=COLTYPE) if QUALITYCONTROL_ROOT: add_analysis_qc_upload_tasks(workflow["stages"], args.productionTag, args.run, "passMC") diff --git a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json new file mode 100644 index 000000000..4823f8664 --- /dev/null +++ b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json @@ -0,0 +1,568 @@ +{ + "EMCClusterMonitorTask": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "0", + "processCollisions": "1", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "1", + "processCollisions": "0", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "ambiguous-track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCentral": "false", + "processMFT": "true" + }, + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "cell-monitor": { + "maxCellTimeMain": "100", + "minCellAmplitude": "0", + "minCellAmplitudeTimeHists": "0.3", + "minCellTimeMain": "-50", + "selectBCID": "all", + "vetoBCID": "" + }, + "emc-tmmonitor": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "hasPropagatedTracks": "1", + "maxTime": "20", + "minM02": "0.1", + "minTime": "-25", + "processCollisions": "1", + "selectBCID": "all", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + }, + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "usePionRejection": "0", + "vertexCut": "-1", + "vetoBCID": "" + }, + "emcal-correction-task": { + "clusterDefinition": "kV3Default", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellFraction": "0.97", + "exoticCellInCrossMinAmplitude": "0.1", + "exoticCellMinAmplitude": "4", + "hasPropagatedTracks": "0", + "hasShaperCorrection": "1", + "isMC": "0", + "logWeight": "4.5", + "maxMatchingDistance": "0.4", + "nonlinearityFunction": "DATA_TestbeamFinal_NoScale", + "processFull": "1", + "processMCFull": "0", + "processStandalone": "0", + "selectedCellType": "1", + "useWeightExotic": "0" + }, + "event-selection-qa-task": { + "isMC": "false", + "isLowFlux" : "false", + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-task": { + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "PbPb" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "-1", + "dQAK0ShortMassWindow": "0.005", + "dQALambdaMassWindow": "0.005", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "1", + "d_UseAutodetectMode": "0", + "d_UseWeightedPCA": "0", + "d_bz": "-999", + "d_doQA": "0", + "d_doTrackQA": "0", + "dcanegtopv": "0.1", + "dcapostopv": "0.1", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "0", + "processRun3": "1", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "2", + "v0cospa": "0.995", + "v0radius": "1" + }, + "multiplicity-table": { + "processRun2": "false", + "processRun3": "true" + }, + "perf-k0s-resolution": { + "etaBins": { + "values": [ + "20", + "-1", + "1" + ] + }, + "eventSelection": "true", + "mBins": { + "values": [ + "300", + "0.35", + "0.65" + ] + }, + "nSigTPC": "10", + "pTBins": { + "values": [ + "200", + "0", + "10" + ] + }, + "phiBins": { + "values": [ + "54", + "0", + "6.2831853" + ] + }, + "rapidity": "0.5", + "requireTRDneg": "0", + "requireTRDpos": "0", + "v0lifetime": "3", + "v0setting_cospa": "0.995", + "v0setting_dcanegtopv": "0.1", + "v0setting_dcapostopv": "0.1", + "v0setting_dcav0dau": "1", + "v0setting_radius": "0.9" + }, + "pid-multiplicity": { + "processIU": "1", + "processStandard": "0" + }, + "qa-efficiency": { + "applyEvSel": "2", + "do-al": "false", + "do-de": "false", + "do-el": "false", + "do-he": "false", + "do-ka": "false", + "do-mu": "false", + "do-pi": "false", + "do-pr": "false", + "do-tr": "false", + "doNegativePDG": "true", + "doPositivePDG": "true", + "doPtEta": "false", + "etaBins": { + "values": [ + "200", + "-0.8", + "0.8" + ] + }, + "globalTrackSelection": "4", + "log-pt": "0", + "make-eff": "true", + "maxProdRadius": "9999", + "nMinNumberOfContributors": "2", + "noFakesHits": "false", + "phiBins": { + "values": [ + "200", + "0", + "6.2839999198913574" + ] + }, + "processData": "true", + "processMC": "false", + "processMCWithoutCollisions": "false", + "ptBins": { + "values": [ + "200", + "0", + "5" + ] + }, + "trackSelection": "true", + "vertex-z-max": "10", + "vertex-z-min": "-10", + "yBins": { + "values": [ + "200", + "-0.5", + "0.5" + ] + } + }, + "qa-event-track": { + "binsPt": { + "values": [ + "0", + "0", + "0.10000000000000001", + "0.20000000000000001", + "0.29999999999999999", + "0.40000000000000002", + "0.5", + "0.59999999999999998", + "0.69999999999999996", + "0.80000000000000004", + "0.90000000000000002", + "1", + "1.1000000000000001", + "1.2", + "1.3", + "1.3999999999999999", + "1.5", + "2", + "5", + "10", + "20", + "50" + ] + }, + "binsTrackMultiplcity": { + "values": [ + "200", + "0", + "200" + ] + }, + "binsVertexPosXY": { + "values": [ + "500", + "-1", + "1" + ] + }, + "binsVertexPosZ": { + "values": [ + "100", + "-20", + "20" + ] + }, + "fractionOfSampledEvents": "1", + "isRun3": "true", + "maxEta": "2", + "maxPhi": "10", + "maxPt": "1e+10", + "minEta": "-2", + "minPhi": "-1", + "minPt": "-10", + "processData": "true", + "processDataIU": "false", + "processDataIUFiltered": "false", + "processMC": "false", + "processTableData": "false", + "processTableMC": "false", + "selectCharge": "0", + "selectGoodEvents": "true", + "selectMaxVtxZ": "100", + "selectPID": "0", + "selectPrim": "false", + "selectSec": "false", + "targetNumberOfEvents": "10000000", + "trackSelection": "1" + }, + "qa-k0s-tracking-efficiency": { + "eventSelection": "true", + "nSigTPC": "10", + "nSigmaBins": { + "values": [ + "1000", + "-100", + "100" + ] + }, + "processIU": "true", + "rapidity": "0.5", + "v0cospa": "0.995" + }, + "tof-signal": "", + "tpc-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "dEdxBins": { + "values": [ + 3000, + 0, + 1000 + ] + }, + "applyRapidityCut": "0", + "processFullWithTOFDeuteron": "0", + "phiBins": { + "values": [ + 50, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "1", + "processKaon": "0", + "processAlpha": "0", + "processFullWithTOFPion": "0", + "processFullHelium3": "1", + "minP": "0.01", + "enableDeDxPlot": "1", + "processFullWithTOFElectron": "0", + "logAxis": "1", + "processFullWithTOFAlpha": "0", + "processFullKaon": "1", + "processMuon": "0", + "processFullDeuteron": "1", + "processFullWithTOFMuon": "0", + "processFullWithTOFProton": "0", + "maxP": "5", + "processFullAlpha": "0", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "processFullWithTOFKaon": "0", + "minTPCNcls": "0", + "processFullMuon": "1", + "processFullWithTOFTriton": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullWithTOFHelium3": "0", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 100, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tof-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "applyRapidityCut": "1", + "phiBins": { + "values": [ + 25, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "0", + "processKaon": "0", + "processAlpha": "0", + "processFullHelium3": "1", + "minP": "0.1", + "logAxis": "0", + "processFullKaon": "1", + "produceDeltaTEtaPhiMap": "0", + "processMuon": "0", + "processFullDeuteron": "1", + "maxP": "5", + "processFullAlpha": "0", + "ptDeltaTEtaPhiMap": "3", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", + "processFullMuon": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 200, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" + }, + "track-extension": { + "processRun2": "false", + "processRun3": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "dcaSetup": "0", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "-1", + "produceTable": "-1", + "ptMax": "1e+10", + "ptMin": "0.1" + } +} diff --git a/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json b/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json similarity index 100% rename from MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json rename to MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json diff --git a/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json new file mode 100644 index 000000000..52c800a8b --- /dev/null +++ b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json @@ -0,0 +1,106 @@ +{ + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-task": { + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "qa-event-track": { + "binsPt": { + "values": [ + "0", + "0", + "0.10000000000000001", + "0.20000000000000001", + "0.29999999999999999", + "0.40000000000000002", + "0.5", + "0.59999999999999998", + "0.69999999999999996", + "0.80000000000000004", + "0.90000000000000002", + "1", + "1.1000000000000001", + "1.2", + "1.3", + "1.3999999999999999", + "1.5", + "2", + "5", + "10", + "20", + "50" + ] + }, + "binsTrackMultiplcity": { + "values": [ + "200", + "0", + "200" + ] + }, + "binsVertexPosXY": { + "values": [ + "500", + "-1", + "1" + ] + }, + "binsVertexPosZ": { + "values": [ + "100", + "-20", + "20" + ] + }, + "fractionOfSampledEvents": "1", + "isRun3": "true", + "maxEta": "2", + "maxPhi": "10", + "maxPt": "1e+10", + "minEta": "-2", + "minPhi": "-1", + "minPt": "-10", + "processData": "true", + "processDataIU": "true", + "processDataIUFiltered": "true", + "processMC": "false", + "processTableData": "false", + "processTableMC": "false", + "selectCharge": "0", + "selectGoodEvents": "true", + "selectMaxVtxZ": "100", + "selectPID": "0", + "selectPrim": "false", + "selectSec": "false", + "targetNumberOfEvents": "10000000", + "trackSelection": "1" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "minPropagationDistance": "5", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "0", + "dcaSetup": "0", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "1", + "itsMatching": "1", + "produceFBextendedTable": "-1", + "produceTable": "1", + "ptMax": "1e+10", + "ptMin": "0.1" + } +} diff --git a/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json b/MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-data.json similarity index 100% rename from MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json rename to MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-data.json diff --git a/MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-mc.json b/MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-mc.json new file mode 100644 index 000000000..5a73aeb23 --- /dev/null +++ b/MC/config/analysis_testing/json/TPCSkimming/pbpb/analysis-testing-mc.json @@ -0,0 +1,315 @@ +{ + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true", + "triggerBcShift": "999" + }, + "event-selection-task": { + "customDeltaBC": "300", + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "0", + "dQAK0ShortMassWindow": "0.00499999989", + "dQALambdaMassWindow": "0.00499999989", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "true", + "d_UseAutodetectMode": "false", + "d_UseWeightedPCA": "false", + "d_bz": "-999", + "d_doQA": "false", + "d_doTrackQA": "false", + "dcanegtopv": "0.100000001", + "dcapostopv": "0.100000001", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "false", + "processRun3": "true", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "0", + "v0cospa": "0.995", + "v0radius": "0.899999976" + }, + "lambdakzero-initializer": "", + "lambdakzero-preselector": { + "dIfMCgenerateAntiHypertriton": "false", + "dIfMCgenerateAntiLambda": "false", + "dIfMCgenerateGamma": "false", + "dIfMCgenerateHypertriton": "false", + "dIfMCgenerateK0Short": "false", + "dIfMCgenerateLambda": "false", + "dPreselectOnlyBaryons": "false", + "dTPCNCrossedRows": "50", + "ddEdxPreSelectAntiHypertriton": "false", + "ddEdxPreSelectAntiLambda": "true", + "ddEdxPreSelectGamma": "true", + "ddEdxPreSelectHypertriton": "false", + "ddEdxPreSelectK0Short": "true", + "ddEdxPreSelectLambda": "true", + "ddEdxPreSelectionWindow": "7", + "processBuildAll": "true", + "processBuildMCAssociated": "false", + "processBuildValiddEdx": "false", + "processBuildValiddEdxMCAssociated": "false" + }, + "lambdakzero-v0-data-link-builder": "", + "multiplicity-table": { + "doVertexZeq": "1", + "processRun2": "false", + "processRun3": "true" + }, + "pid-multiplicity": { + "processIU": "false", + "processStandard": "true" + }, + "timestamp-task": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "isRun2MC": "false", + "orbit-reset-path": "CTP/Calib/OrbitReset", + "rct-path": "RCT/Info/RunInformation", + "verbose": "false" + }, + "tof-event-time": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "maxEvTimeTOF": "100000", + "maxMomentum": "2", + "minMomentum": "0.5", + "param-file": "", + "param-sigma": "TOFReso", + "processFT0": "false", + "processNoFT0": "true", + "processOnlyFT0": "false", + "processRun2": "false" + }, + "tof-pid-beta": { + "tof-expreso": "80" + }, + "tof-pid-full": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "enableTimeDependentResponse": "false", + "fatalOnPassNotAvailable": "true", + "param-file": "", + "param-sigma": "TOFResoParams", + "passName": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "processWSlice": "true", + "processWoSlice": "false", + "processWoSliceDev": "false", + "useParamCollection": "false" + }, + "tof-signal": { + "processRun2": "false", + "processRun3": "true" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://ccdb-test.cern.ch:8080", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "false", + "dEdxBins": { + "values": [ + "5000", + "0", + "5000" + ] + }, + "deltaBins": { + "values": [ + "200", + "-1000", + "1000" + ] + }, + "enableDeDxPlot": "true", + "etaBins": { + "values": [ + "100", + "-1", + "1" + ] + }, + "expSigmaBins": { + "values": [ + "200", + "0", + "200" + ] + }, + "logAxis": "1", + "maxP": "20", + "minP": "0.100000001", + "minTPCNcls": "0", + "nBinsP": "200", + "nSigmaBins": { + "values": [ + "401", + "-10.024999618530273", + "10.024999618530273" + ] + }, + "phiBins": { + "values": [ + "100", + "0", + "6.283185307179586" + ] + }, + "processAlpha": "false", + "processDeuteron": "false", + "processElectron": "false", + "processFullAlpha": "true", + "processFullDeuteron": "true", + "processFullElectron": "true", + "processFullHelium3": "true", + "processFullKaon": "true", + "processFullMuon": "true", + "processFullPion": "true", + "processFullProton": "true", + "processFullTriton": "true", + "processFullWithTOFAlpha": "false", + "processFullWithTOFDeuteron": "false", + "processFullWithTOFElectron": "false", + "processFullWithTOFHelium3": "false", + "processFullWithTOFKaon": "false", + "processFullWithTOFMuon": "false", + "processFullWithTOFPion": "false", + "processFullWithTOFProton": "false", + "processFullWithTOFTriton": "false", + "processHelium3": "false", + "processKaon": "false", + "processMuon": "false", + "processPion": "false", + "processProton": "false", + "processTriton": "false", + "splitSignalPerCharge": "true", + "trackLengthBins": { + "values": [ + "100", + "0", + "1000" + ] + }, + "trackSelection": "1" + }, + "track-pid-qa": { + "dcamax": "1e+10", + "dcamin": "0", + "maxchi2tpc": "4", + "mincrossedrows": "70", + "processDummy": "true", + "processQA": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "minPropagationDistance": "2.5", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "0", + "ptMax": "1e+10", + "ptMin": "0.1" + }, + "tree-writer-t-p-c-t-o-f": { + "applyEvSel": "2", + "applyTrkSel": "1", + "downsamplingTsalisKaons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_Ka": "4.0000000000000003e-05", + "dwnSmplFactor_Pi": "4.0000000000000003e-05", + "dwnSmplFactor_Pr": "0.00014999999999999999", + "maxMomTPCOnlyKa": "0.5", + "maxMomTPCOnlyPi": "0.5", + "maxMomTPCOnlyPr": "0.600000024", + "nClNorm": "152", + "nSigmaTOF_TPCTOF_Ka": "4", + "nSigmaTOF_TPCTOF_Pi": "4", + "nSigmaTOF_TPCTOF_Pr": "4", + "nSigmaTPCOnlyKa": "4", + "nSigmaTPCOnlyPi": "4", + "nSigmaTPCOnlyPr": "4", + "nSigmaTPC_TPCTOF_Ka": "5", + "nSigmaTPC_TPCTOF_Pi": "5", + "nSigmaTPC_TPCTOF_Pr": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "tree-writer-tpc-v0": { + "applyEvSel": "2", + "downsamplingTsalisElectrons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_El": "0.01", + "dwnSmplFactor_Pi": "0.001", + "dwnSmplFactor_Pr": "0.02", + "nClNorm": "152", + "nSigmaTOFdautrack": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "v0-selector": { + "dcamax": "1e+10", + "dcamin": "0", + "dcav0dau": "0.300000012", + "maxchi2tpc": "4", + "maxpsipair": "1.60000002", + "mincrossedrows": "70", + "v0Rmax": "90", + "v0Rmin": "0", + "v0cospa": "0.998000026", + "v0max_mee": "0.100000001" + } +} diff --git a/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-data.json new file mode 100644 index 000000000..5a73aeb23 --- /dev/null +++ b/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-data.json @@ -0,0 +1,315 @@ +{ + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true", + "triggerBcShift": "999" + }, + "event-selection-task": { + "customDeltaBC": "300", + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "0", + "dQAK0ShortMassWindow": "0.00499999989", + "dQALambdaMassWindow": "0.00499999989", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "true", + "d_UseAutodetectMode": "false", + "d_UseWeightedPCA": "false", + "d_bz": "-999", + "d_doQA": "false", + "d_doTrackQA": "false", + "dcanegtopv": "0.100000001", + "dcapostopv": "0.100000001", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "false", + "processRun3": "true", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "0", + "v0cospa": "0.995", + "v0radius": "0.899999976" + }, + "lambdakzero-initializer": "", + "lambdakzero-preselector": { + "dIfMCgenerateAntiHypertriton": "false", + "dIfMCgenerateAntiLambda": "false", + "dIfMCgenerateGamma": "false", + "dIfMCgenerateHypertriton": "false", + "dIfMCgenerateK0Short": "false", + "dIfMCgenerateLambda": "false", + "dPreselectOnlyBaryons": "false", + "dTPCNCrossedRows": "50", + "ddEdxPreSelectAntiHypertriton": "false", + "ddEdxPreSelectAntiLambda": "true", + "ddEdxPreSelectGamma": "true", + "ddEdxPreSelectHypertriton": "false", + "ddEdxPreSelectK0Short": "true", + "ddEdxPreSelectLambda": "true", + "ddEdxPreSelectionWindow": "7", + "processBuildAll": "true", + "processBuildMCAssociated": "false", + "processBuildValiddEdx": "false", + "processBuildValiddEdxMCAssociated": "false" + }, + "lambdakzero-v0-data-link-builder": "", + "multiplicity-table": { + "doVertexZeq": "1", + "processRun2": "false", + "processRun3": "true" + }, + "pid-multiplicity": { + "processIU": "false", + "processStandard": "true" + }, + "timestamp-task": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "isRun2MC": "false", + "orbit-reset-path": "CTP/Calib/OrbitReset", + "rct-path": "RCT/Info/RunInformation", + "verbose": "false" + }, + "tof-event-time": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "maxEvTimeTOF": "100000", + "maxMomentum": "2", + "minMomentum": "0.5", + "param-file": "", + "param-sigma": "TOFReso", + "processFT0": "false", + "processNoFT0": "true", + "processOnlyFT0": "false", + "processRun2": "false" + }, + "tof-pid-beta": { + "tof-expreso": "80" + }, + "tof-pid-full": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "enableTimeDependentResponse": "false", + "fatalOnPassNotAvailable": "true", + "param-file": "", + "param-sigma": "TOFResoParams", + "passName": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "processWSlice": "true", + "processWoSlice": "false", + "processWoSliceDev": "false", + "useParamCollection": "false" + }, + "tof-signal": { + "processRun2": "false", + "processRun3": "true" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://ccdb-test.cern.ch:8080", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "false", + "dEdxBins": { + "values": [ + "5000", + "0", + "5000" + ] + }, + "deltaBins": { + "values": [ + "200", + "-1000", + "1000" + ] + }, + "enableDeDxPlot": "true", + "etaBins": { + "values": [ + "100", + "-1", + "1" + ] + }, + "expSigmaBins": { + "values": [ + "200", + "0", + "200" + ] + }, + "logAxis": "1", + "maxP": "20", + "minP": "0.100000001", + "minTPCNcls": "0", + "nBinsP": "200", + "nSigmaBins": { + "values": [ + "401", + "-10.024999618530273", + "10.024999618530273" + ] + }, + "phiBins": { + "values": [ + "100", + "0", + "6.283185307179586" + ] + }, + "processAlpha": "false", + "processDeuteron": "false", + "processElectron": "false", + "processFullAlpha": "true", + "processFullDeuteron": "true", + "processFullElectron": "true", + "processFullHelium3": "true", + "processFullKaon": "true", + "processFullMuon": "true", + "processFullPion": "true", + "processFullProton": "true", + "processFullTriton": "true", + "processFullWithTOFAlpha": "false", + "processFullWithTOFDeuteron": "false", + "processFullWithTOFElectron": "false", + "processFullWithTOFHelium3": "false", + "processFullWithTOFKaon": "false", + "processFullWithTOFMuon": "false", + "processFullWithTOFPion": "false", + "processFullWithTOFProton": "false", + "processFullWithTOFTriton": "false", + "processHelium3": "false", + "processKaon": "false", + "processMuon": "false", + "processPion": "false", + "processProton": "false", + "processTriton": "false", + "splitSignalPerCharge": "true", + "trackLengthBins": { + "values": [ + "100", + "0", + "1000" + ] + }, + "trackSelection": "1" + }, + "track-pid-qa": { + "dcamax": "1e+10", + "dcamin": "0", + "maxchi2tpc": "4", + "mincrossedrows": "70", + "processDummy": "true", + "processQA": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "minPropagationDistance": "2.5", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "0", + "ptMax": "1e+10", + "ptMin": "0.1" + }, + "tree-writer-t-p-c-t-o-f": { + "applyEvSel": "2", + "applyTrkSel": "1", + "downsamplingTsalisKaons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_Ka": "4.0000000000000003e-05", + "dwnSmplFactor_Pi": "4.0000000000000003e-05", + "dwnSmplFactor_Pr": "0.00014999999999999999", + "maxMomTPCOnlyKa": "0.5", + "maxMomTPCOnlyPi": "0.5", + "maxMomTPCOnlyPr": "0.600000024", + "nClNorm": "152", + "nSigmaTOF_TPCTOF_Ka": "4", + "nSigmaTOF_TPCTOF_Pi": "4", + "nSigmaTOF_TPCTOF_Pr": "4", + "nSigmaTPCOnlyKa": "4", + "nSigmaTPCOnlyPi": "4", + "nSigmaTPCOnlyPr": "4", + "nSigmaTPC_TPCTOF_Ka": "5", + "nSigmaTPC_TPCTOF_Pi": "5", + "nSigmaTPC_TPCTOF_Pr": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "tree-writer-tpc-v0": { + "applyEvSel": "2", + "downsamplingTsalisElectrons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_El": "0.01", + "dwnSmplFactor_Pi": "0.001", + "dwnSmplFactor_Pr": "0.02", + "nClNorm": "152", + "nSigmaTOFdautrack": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "v0-selector": { + "dcamax": "1e+10", + "dcamin": "0", + "dcav0dau": "0.300000012", + "maxchi2tpc": "4", + "maxpsipair": "1.60000002", + "mincrossedrows": "70", + "v0Rmax": "90", + "v0Rmin": "0", + "v0cospa": "0.998000026", + "v0max_mee": "0.100000001" + } +} diff --git a/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-mc.json new file mode 100644 index 000000000..5a73aeb23 --- /dev/null +++ b/MC/config/analysis_testing/json/TPCSkimming/pp/analysis-testing-mc.json @@ -0,0 +1,315 @@ +{ + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true", + "triggerBcShift": "999" + }, + "event-selection-task": { + "customDeltaBC": "300", + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "0", + "dQAK0ShortMassWindow": "0.00499999989", + "dQALambdaMassWindow": "0.00499999989", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "true", + "d_UseAutodetectMode": "false", + "d_UseWeightedPCA": "false", + "d_bz": "-999", + "d_doQA": "false", + "d_doTrackQA": "false", + "dcanegtopv": "0.100000001", + "dcapostopv": "0.100000001", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "false", + "processRun3": "true", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "0", + "v0cospa": "0.995", + "v0radius": "0.899999976" + }, + "lambdakzero-initializer": "", + "lambdakzero-preselector": { + "dIfMCgenerateAntiHypertriton": "false", + "dIfMCgenerateAntiLambda": "false", + "dIfMCgenerateGamma": "false", + "dIfMCgenerateHypertriton": "false", + "dIfMCgenerateK0Short": "false", + "dIfMCgenerateLambda": "false", + "dPreselectOnlyBaryons": "false", + "dTPCNCrossedRows": "50", + "ddEdxPreSelectAntiHypertriton": "false", + "ddEdxPreSelectAntiLambda": "true", + "ddEdxPreSelectGamma": "true", + "ddEdxPreSelectHypertriton": "false", + "ddEdxPreSelectK0Short": "true", + "ddEdxPreSelectLambda": "true", + "ddEdxPreSelectionWindow": "7", + "processBuildAll": "true", + "processBuildMCAssociated": "false", + "processBuildValiddEdx": "false", + "processBuildValiddEdxMCAssociated": "false" + }, + "lambdakzero-v0-data-link-builder": "", + "multiplicity-table": { + "doVertexZeq": "1", + "processRun2": "false", + "processRun3": "true" + }, + "pid-multiplicity": { + "processIU": "false", + "processStandard": "true" + }, + "timestamp-task": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "isRun2MC": "false", + "orbit-reset-path": "CTP/Calib/OrbitReset", + "rct-path": "RCT/Info/RunInformation", + "verbose": "false" + }, + "tof-event-time": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "maxEvTimeTOF": "100000", + "maxMomentum": "2", + "minMomentum": "0.5", + "param-file": "", + "param-sigma": "TOFReso", + "processFT0": "false", + "processNoFT0": "true", + "processOnlyFT0": "false", + "processRun2": "false" + }, + "tof-pid-beta": { + "tof-expreso": "80" + }, + "tof-pid-full": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TOF", + "enableTimeDependentResponse": "false", + "fatalOnPassNotAvailable": "true", + "param-file": "", + "param-sigma": "TOFResoParams", + "passName": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "processWSlice": "true", + "processWoSlice": "false", + "processWoSliceDev": "false", + "useParamCollection": "false" + }, + "tof-signal": { + "processRun2": "false", + "processRun3": "true" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://ccdb-test.cern.ch:8080", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "false", + "dEdxBins": { + "values": [ + "5000", + "0", + "5000" + ] + }, + "deltaBins": { + "values": [ + "200", + "-1000", + "1000" + ] + }, + "enableDeDxPlot": "true", + "etaBins": { + "values": [ + "100", + "-1", + "1" + ] + }, + "expSigmaBins": { + "values": [ + "200", + "0", + "200" + ] + }, + "logAxis": "1", + "maxP": "20", + "minP": "0.100000001", + "minTPCNcls": "0", + "nBinsP": "200", + "nSigmaBins": { + "values": [ + "401", + "-10.024999618530273", + "10.024999618530273" + ] + }, + "phiBins": { + "values": [ + "100", + "0", + "6.283185307179586" + ] + }, + "processAlpha": "false", + "processDeuteron": "false", + "processElectron": "false", + "processFullAlpha": "true", + "processFullDeuteron": "true", + "processFullElectron": "true", + "processFullHelium3": "true", + "processFullKaon": "true", + "processFullMuon": "true", + "processFullPion": "true", + "processFullProton": "true", + "processFullTriton": "true", + "processFullWithTOFAlpha": "false", + "processFullWithTOFDeuteron": "false", + "processFullWithTOFElectron": "false", + "processFullWithTOFHelium3": "false", + "processFullWithTOFKaon": "false", + "processFullWithTOFMuon": "false", + "processFullWithTOFPion": "false", + "processFullWithTOFProton": "false", + "processFullWithTOFTriton": "false", + "processHelium3": "false", + "processKaon": "false", + "processMuon": "false", + "processPion": "false", + "processProton": "false", + "processTriton": "false", + "splitSignalPerCharge": "true", + "trackLengthBins": { + "values": [ + "100", + "0", + "1000" + ] + }, + "trackSelection": "1" + }, + "track-pid-qa": { + "dcamax": "1e+10", + "dcamin": "0", + "maxchi2tpc": "4", + "mincrossedrows": "70", + "processDummy": "true", + "processQA": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "minPropagationDistance": "2.5", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "0", + "ptMax": "1e+10", + "ptMin": "0.1" + }, + "tree-writer-t-p-c-t-o-f": { + "applyEvSel": "2", + "applyTrkSel": "1", + "downsamplingTsalisKaons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_Ka": "4.0000000000000003e-05", + "dwnSmplFactor_Pi": "4.0000000000000003e-05", + "dwnSmplFactor_Pr": "0.00014999999999999999", + "maxMomTPCOnlyKa": "0.5", + "maxMomTPCOnlyPi": "0.5", + "maxMomTPCOnlyPr": "0.600000024", + "nClNorm": "152", + "nSigmaTOF_TPCTOF_Ka": "4", + "nSigmaTOF_TPCTOF_Pi": "4", + "nSigmaTOF_TPCTOF_Pr": "4", + "nSigmaTPCOnlyKa": "4", + "nSigmaTPCOnlyPi": "4", + "nSigmaTPCOnlyPr": "4", + "nSigmaTPC_TPCTOF_Ka": "5", + "nSigmaTPC_TPCTOF_Pi": "5", + "nSigmaTPC_TPCTOF_Pr": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "tree-writer-tpc-v0": { + "applyEvSel": "2", + "downsamplingTsalisElectrons": "-1", + "downsamplingTsalisPions": "-1", + "downsamplingTsalisProtons": "-1", + "dwnSmplFactor_El": "0.01", + "dwnSmplFactor_Pi": "0.001", + "dwnSmplFactor_Pr": "0.02", + "nClNorm": "152", + "nSigmaTOFdautrack": "5", + "sqrt_s_NN": "0", + "trackSelection": "1" + }, + "v0-selector": { + "dcamax": "1e+10", + "dcamin": "0", + "dcav0dau": "0.300000012", + "maxchi2tpc": "4", + "maxpsipair": "1.60000002", + "mincrossedrows": "70", + "v0Rmax": "90", + "v0Rmin": "0", + "v0cospa": "0.998000026", + "v0max_mee": "0.100000001" + } +} diff --git a/MC/config/analysis_testing/json/analyses_config.json b/MC/config/analysis_testing/json/analyses_config.json index a82a588ab..9c060eaa1 100644 --- a/MC/config/analysis_testing/json/analyses_config.json +++ b/MC/config/analysis_testing/json/analyses_config.json @@ -4,9 +4,8 @@ "name": "MCHistograms", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysistutorial-mc-histograms"] @@ -15,10 +14,8 @@ "name": "Efficiency", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-trackselection", @@ -29,10 +26,8 @@ "name": "EventTrackQA", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-EventTrackQA-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-trackselection", @@ -43,10 +38,8 @@ "name": "K0STrackingEfficiencyQA", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-lf-lambdakzerobuilder", "o2-analysis-track-propagation", "o2-analysis-trackselection", @@ -65,10 +58,8 @@ "name": "Validation", "enabled": false, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-validation"] @@ -77,10 +68,8 @@ "name": "PIDFull", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-ft0-corrected-table", "o2-analysis-timestamp", "o2-analysis-track-propagation", @@ -104,10 +93,8 @@ "AnalysisResults.root", "AnalysisResults_trees.root" ], - "config": { - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json", - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-TPCSkimming.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": [ "o2-analysis-pid-tof-full", "o2-analysis-pid-tof-beta", @@ -129,10 +116,8 @@ "name": "PWGMMMFT", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-trackselection", @@ -145,10 +130,8 @@ "name": "EventSelectionQA", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json", - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-event-selection", @@ -158,9 +141,8 @@ "name": "WeakDecayTutorial", "enabled": false, "expected_output": null, - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysistutorial-weak-decay-iteration"] @@ -169,18 +151,16 @@ "name": "CheckDataModelMC", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-check-data-model-mc"] }, { "name": "LK0CFFemto", "enabled": false, "expected_output": ["AnalysisResults.root", "QAResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-multiplicity-table --aod-writer-json aodWriterTempConfig.json", "o2-analysis-timestamp", "o2-analysis-track-propagation", @@ -196,9 +176,8 @@ "name": "PWGMMFwdVertexing", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-timestamp", "o2-analysis-mm-vertexing-fwd"] }, @@ -206,9 +185,8 @@ "name": "MCSimpleValidation", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-trackselection", @@ -219,9 +197,8 @@ "name": "PWGMMMDnDeta", "enabled": false, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-event-selection", @@ -232,9 +209,8 @@ "name": "PWGHFD0", "enabled": false, "expected_output": ["AnalysisResults.root"], - "config": { - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": false, "tasks": ["o2-analysis-hf-track-index-skims-creator", "o2-analysis-hf-candidate-creator-2prong", "o2-analysis-hf-d0-candidate-selector", @@ -253,10 +229,8 @@ "name": "EMCAL", "enabled": true, "expected_output": ["AnalysisResults.root"], - "config" : { - "data": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-data.json", - "mc": "json://${O2DPG_ROOT}/MC/config/analysis_testing/json/analysis-testing-mc.json" - }, + "valid_mc": true, + "valid_data": true, "tasks": ["o2-analysis-je-emc-eventselection-qa", "o2-analysis-je-emc-cellmonitor", "o2-analysis-je-emcal-correction-task", @@ -265,7 +239,7 @@ "o2-analysis-timestamp", "o2-analysis-track-propagation", "o2-analysis-trackselection", - "o2-analysis-event-selection", + "o2-analysis-event-selection", "o2-analysis-pid-tpc-full", "o2-analysis-pid-tpc-base" ] diff --git a/MC/config/analysis_testing/json/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json similarity index 100% rename from MC/config/analysis_testing/json/analysis-testing-data.json rename to MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json diff --git a/MC/config/analysis_testing/json/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json similarity index 100% rename from MC/config/analysis_testing/json/analysis-testing-mc.json rename to MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json new file mode 100644 index 000000000..6b1ad0ad5 --- /dev/null +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -0,0 +1,567 @@ +{ + "EMCClusterMonitorTask": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "0", + "processCollisions": "1", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "1", + "processCollisions": "0", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "ambiguous-track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCentral": "false", + "processMFT": "true" + }, + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "cell-monitor": { + "maxCellTimeMain": "100", + "minCellAmplitude": "0", + "minCellAmplitudeTimeHists": "0.3", + "minCellTimeMain": "-50", + "selectBCID": "all", + "vetoBCID": "" + }, + "emc-tmmonitor": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "hasPropagatedTracks": "1", + "maxTime": "20", + "minM02": "0.1", + "minTime": "-25", + "processCollisions": "1", + "selectBCID": "all", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + }, + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "usePionRejection": "0", + "vertexCut": "-1", + "vetoBCID": "" + }, + "emcal-correction-task": { + "clusterDefinition": "kV3Default", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellFraction": "0.97", + "exoticCellInCrossMinAmplitude": "0.1", + "exoticCellMinAmplitude": "4", + "hasPropagatedTracks": "0", + "hasShaperCorrection": "1", + "isMC": "0", + "logWeight": "4.5", + "maxMatchingDistance": "0.4", + "nonlinearityFunction": "DATA_TestbeamFinal_NoScale", + "processFull": "1", + "processMCFull": "0", + "processStandalone": "0", + "selectedCellType": "1", + "useWeightExotic": "0" + }, + "event-selection-qa-task": { + "isMC": "false", + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-task": { + "isMC": "false", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "-1", + "dQAK0ShortMassWindow": "0.005", + "dQALambdaMassWindow": "0.005", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "1", + "d_UseAutodetectMode": "0", + "d_UseWeightedPCA": "0", + "d_bz": "-999", + "d_doQA": "0", + "d_doTrackQA": "0", + "dcanegtopv": "0.1", + "dcapostopv": "0.1", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "0", + "processRun3": "1", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "2", + "v0cospa": "0.995", + "v0radius": "1" + }, + "multiplicity-table": { + "processRun2": "false", + "processRun3": "true" + }, + "perf-k0s-resolution": { + "etaBins": { + "values": [ + "20", + "-1", + "1" + ] + }, + "eventSelection": "true", + "mBins": { + "values": [ + "300", + "0.35", + "0.65" + ] + }, + "nSigTPC": "10", + "pTBins": { + "values": [ + "200", + "0", + "10" + ] + }, + "phiBins": { + "values": [ + "54", + "0", + "6.2831853" + ] + }, + "rapidity": "0.5", + "requireTRDneg": "0", + "requireTRDpos": "0", + "v0lifetime": "3", + "v0setting_cospa": "0.995", + "v0setting_dcanegtopv": "0.1", + "v0setting_dcapostopv": "0.1", + "v0setting_dcav0dau": "1", + "v0setting_radius": "0.9" + }, + "pid-multiplicity": { + "processIU": "1", + "processStandard": "0" + }, + "qa-efficiency": { + "applyEvSel": "2", + "do-al": "false", + "do-de": "false", + "do-el": "false", + "do-he": "false", + "do-ka": "false", + "do-mu": "false", + "do-pi": "false", + "do-pr": "false", + "do-tr": "false", + "doNegativePDG": "true", + "doPositivePDG": "true", + "doPtEta": "false", + "etaBins": { + "values": [ + "200", + "-0.8", + "0.8" + ] + }, + "globalTrackSelection": "4", + "log-pt": "0", + "make-eff": "true", + "maxProdRadius": "9999", + "nMinNumberOfContributors": "2", + "noFakesHits": "false", + "phiBins": { + "values": [ + "200", + "0", + "6.2839999198913574" + ] + }, + "processData": "true", + "processMC": "false", + "processMCWithoutCollisions": "false", + "ptBins": { + "values": [ + "200", + "0", + "5" + ] + }, + "trackSelection": "true", + "vertex-z-max": "10", + "vertex-z-min": "-10", + "yBins": { + "values": [ + "200", + "-0.5", + "0.5" + ] + } + }, + "qa-event-track": { + "binsPt": { + "values": [ + "0", + "0", + "0.10000000000000001", + "0.20000000000000001", + "0.29999999999999999", + "0.40000000000000002", + "0.5", + "0.59999999999999998", + "0.69999999999999996", + "0.80000000000000004", + "0.90000000000000002", + "1", + "1.1000000000000001", + "1.2", + "1.3", + "1.3999999999999999", + "1.5", + "2", + "5", + "10", + "20", + "50" + ] + }, + "binsTrackMultiplcity": { + "values": [ + "200", + "0", + "200" + ] + }, + "binsVertexPosXY": { + "values": [ + "500", + "-1", + "1" + ] + }, + "binsVertexPosZ": { + "values": [ + "100", + "-20", + "20" + ] + }, + "fractionOfSampledEvents": "1", + "isRun3": "true", + "maxEta": "2", + "maxPhi": "10", + "maxPt": "1e+10", + "minEta": "-2", + "minPhi": "-1", + "minPt": "-10", + "processData": "true", + "processDataIU": "false", + "processDataIUFiltered": "false", + "processMC": "false", + "processTableData": "false", + "processTableMC": "false", + "selectCharge": "0", + "selectGoodEvents": "true", + "selectMaxVtxZ": "100", + "selectPID": "0", + "selectPrim": "false", + "selectSec": "false", + "targetNumberOfEvents": "10000000", + "trackSelection": "1" + }, + "qa-k0s-tracking-efficiency": { + "eventSelection": "true", + "nSigTPC": "10", + "nSigmaBins": { + "values": [ + "1000", + "-100", + "100" + ] + }, + "processIU": "true", + "rapidity": "0.5", + "v0cospa": "0.995" + }, + "tof-signal": "", + "tpc-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "dEdxBins": { + "values": [ + 3000, + 0, + 1000 + ] + }, + "applyRapidityCut": "0", + "processFullWithTOFDeuteron": "0", + "phiBins": { + "values": [ + 50, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "1", + "processKaon": "0", + "processAlpha": "0", + "processFullWithTOFPion": "0", + "processFullHelium3": "1", + "minP": "0.01", + "enableDeDxPlot": "1", + "processFullWithTOFElectron": "0", + "logAxis": "1", + "processFullWithTOFAlpha": "0", + "processFullKaon": "1", + "processMuon": "0", + "processFullDeuteron": "1", + "processFullWithTOFMuon": "0", + "processFullWithTOFProton": "0", + "maxP": "5", + "processFullAlpha": "0", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "processFullWithTOFKaon": "0", + "minTPCNcls": "0", + "processFullMuon": "1", + "processFullWithTOFTriton": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullWithTOFHelium3": "0", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 100, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tof-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "applyRapidityCut": "1", + "phiBins": { + "values": [ + 25, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "0", + "processKaon": "0", + "processAlpha": "0", + "processFullHelium3": "1", + "minP": "0.1", + "logAxis": "0", + "processFullKaon": "1", + "produceDeltaTEtaPhiMap": "0", + "processMuon": "0", + "processFullDeuteron": "1", + "maxP": "5", + "processFullAlpha": "0", + "ptDeltaTEtaPhiMap": "3", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", + "processFullMuon": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 200, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" + }, + "track-extension": { + "processRun2": "false", + "processRun3": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "dcaSetup": "0", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "-1", + "produceTable": "-1", + "ptMax": "1e+10", + "ptMin": "0.1" + } +} \ No newline at end of file diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json new file mode 100644 index 000000000..62209a027 --- /dev/null +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -0,0 +1,1442 @@ +{ + "McCollisions": {}, + "McParticles": {}, + "McTrackLabels": {}, + "ambiguous-track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCentral": "false", + "processMFT": "true" + }, + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "check-mc-particles-indices": { + "debugMode": "0" + }, + "check-mc-particles-indices-grouped": { + "debugMode": "1" + }, + "event-selection-qa-task": { + "isMC": "true", + "processMCRun3": "true", + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-task": { + "isMC": "true", + "muonSelection": "0", + "processRun2": "false", + "processRun3": "true", + "syst": "pp" + }, + "femto-dream-producer-task": { + "ConfDebugOutput": "true", + "ConfEvtOfflineCheck": "false", + "ConfEvtTriggerCheck": "true", + "ConfEvtTriggerSel": "0", + "ConfEvtZvtx": "10", + "ConfIsRun3": "false", + "ConfIsTrigger": "false", + "ConfStoreV0": "true", + "ConfTrkDCAxyMax": { + "values": [ + "0.100000001", + "3.5" + ] + }, + "ConfTrkDCAzMax": { + "values": [ + "0.200000003", + "3.5" + ] + }, + "ConfTrkEtaMax": { + "values": [ + "0.800000012", + "0.699999988", + "0.899999976" + ] + }, + "ConfTrkITSnClsIbMin": { + "values": [ + "-1", + "1" + ] + }, + "ConfTrkITSnClsMin": { + "values": [ + "-1", + "2", + "4" + ] + }, + "ConfTrkPIDnSigmaMax": { + "values": [ + "3.5", + "3", + "2.5" + ] + }, + "ConfTrkPtMin": { + "values": [ + "0.400000006", + "0.600000024", + "0.5" + ] + }, + "ConfTrkSign": { + "values": [ + "-1", + "1" + ] + }, + "ConfTrkTPCcRowsMin": { + "values": [ + "70", + "60", + "80" + ] + }, + "ConfTrkTPCfClsMin": { + "values": [ + "0.699999988", + "0.829999983", + "0.899999976" + ] + }, + "ConfTrkTPCnClsMin": { + "values": [ + "80", + "70", + "60" + ] + }, + "ConfTrkTPCsClsMax": { + "values": [ + "0.100000001", + "160" + ] + }, + "ConfTrkTPIDspecies": { + "values": [ + "2", + "3", + "4", + "5" + ] + }, + "ConfV0CPAMin": { + "values": [ + "0.99000001", + "0.995000005" + ] + }, + "ConfV0DCAdaughMax": { + "values": [ + "1.20000005", + "1.5" + ] + }, + "ConfV0DaughDCAMin": { + "values": [ + "0.0500000007", + "0.0599999987" + ] + }, + "ConfV0DaughPIDnSigmaMax": { + "values": [ + "5", + "4" + ] + }, + "ConfV0DaughTPCnclsMin": { + "values": [ + "80", + "70", + "60" + ] + }, + "ConfV0DecVecMax": { + "values": [ + "100.0" + ] + }, + "ConfV0PtMin": { + "values": [ + "0.300000012", + "0.400000006", + "0.5" + ] + }, + "ConfV0Sign": { + "values": [ + "-1", + "1" + ] + }, + "ConfV0TranRadMax": { + "values": [ + "100.0" + ] + }, + "ConfV0TranRadMin": { + "values": [ + "0.2" + ] + } + }, + "hf-cand-creator-2prong": { + "b_dovalplots": "true", + "b_propdca": "true", + "d_bz": "5.", + "d_maxdzini": "999.", + "d_maxr": "5.", + "d_minparamchange": "0.001", + "d_minrelchi2change": "0.9" + }, + "hf-cand-creator-2prong-expressions": { + "processMC": "true" + }, + "hf-d0-candidate-selector": { + "D0_to_pi_K_cuts": { + "values": [ + [ + "0.4", + "0.035", + "0.8", + "0.5", + "0.5", + "0.1", + "0.1", + "-5.e-5", + "0.80", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.035", + "0.8", + "0.5", + "0.5", + "0.1", + "0.1", + "-5.e-5", + "0.80", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.4", + "0.4", + "0.1", + "0.1", + "-25.e-5", + "0.80", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.4", + "0.4", + "0.1", + "0.1", + "-25.e-5", + "0.80", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-20.e-5", + "0.90", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-20.e-5", + "0.90", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-12.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-12.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-8.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-7.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.8", + "0.7", + "0.7", + "0.1", + "0.1", + "-7.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.9", + "0.7", + "0.7", + "0.1", + "0.1", + "-5.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.9", + "0.7", + "0.7", + "0.1", + "0.1", + "-5.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "0.9", + "0.7", + "0.7", + "0.1", + "0.1", + "-5.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.7", + "0.7", + "0.1", + "0.1", + "10.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.7", + "0.7", + "0.1", + "0.1", + "1000.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.7", + "0.7", + "0.1", + "0.1", + "1000.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.7", + "0.7", + "0.1", + "0.1", + "1000.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.7", + "0.7", + "0.1", + "0.1", + "1000.e-5", + "0.85", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ], + [ + "0.4", + "0.030", + "1.0", + "0.6", + "0.6", + "0.1", + "0.1", + "1000.e-5", + "0.80", + "0.", + "0.", + "10.0", + "10.0", + "0.06" + ] + ] + }, + "d_TPCNClsFindablePIDCut": "50.", + "d_nSigmaTOF": "3.", + "d_nSigmaTOFCombined": "5.", + "d_nSigmaTPC": "3.", + "d_nSigmaTPCCombined": "5.", + "d_pTCandMax": "50.", + "d_pTCandMin": "0.", + "d_pidTOFMaxpT": "10.", + "d_pidTOFMinpT": "0.15", + "d_pidTPCMaxpT": "10.", + "d_pidTPCMinpT": "0.15", + "pTBins": { + "values": [ + "0", + "0.5", + "1", + "1.5", + "2", + "2.5", + "3", + "3.5", + "4", + "4.5", + "5", + "5.5", + "6", + "6.5", + "7", + "7.5", + "8", + "9", + "10", + "12", + "16", + "20", + "24", + "36", + "50", + "100" + ] + } + }, + "hf-task-d0": { + "cutYCandMax": "0.8", + "d_selectionFlagD0": "0", + "d_selectionFlagD0bar": "0", + "pTBins": { + "values": [ + "0", + "0.5", + "1", + "1.5", + "2", + "2.5", + "3", + "3.5", + "4", + "4.5", + "5", + "5.5", + "6", + "6.5", + "7", + "7.5", + "8", + "9", + "10", + "12", + "16", + "20", + "24", + "36", + "50", + "100" + ] + }, + "processMC": "true" + }, + "hf-track-index-skims-creator": { + "bz": "5.", + "cutsD0ToPiK": { + "values": [ + [ + "1.61", + "2.12", + "0.5", + "0.00" + ], + [ + "1.61", + "2.12", + "0.5", + "0.00" + ] + ] + }, + "cutsDPlusToPiKPi": { + "values": [ + [ + "1.67", + "2.07", + "0.7", + "0.03" + ], + [ + "1.67", + "2.07", + "0.7", + "0.03" + ] + ] + }, + "cutsDsToPiKK": { + "values": [ + [ + "1.74", + "2.18", + "0.94", + "0.02" + ], + [ + "1.74", + "2.18", + "0.94", + "0.02" + ] + ] + }, + "cutsJpsiToEE": { + "values": [ + [ + "2.5", + "4.1", + "-2.", + "1000.00" + ], + [ + "2.5", + "4.1", + "-2.", + "1000.00" + ] + ] + }, + "cutsJpsiToMuMu": { + "values": [ + [ + "2.5", + "4.1", + "-2.", + "1000.00" + ], + [ + "2.5", + "4.1", + "-2.", + "1000.00" + ] + ] + }, + "cutsLcToPKPi": { + "values": [ + [ + "1.98", + "2.58", + "-2.", + "0.0" + ], + [ + "1.98", + "2.58", + "-2.", + "0.0" + ] + ] + }, + "cutsXicToPKPi": { + "values": [ + [ + "2.10", + "2.80", + "-2.", + "0.0" + ], + [ + "2.10", + "2.80", + "-2.", + "0.0" + ] + ] + }, + "do3prong": "1", + "fillHistograms": "true", + "maxDZIni": "999.", + "maxRad": "5.", + "minParamChange": "0.001", + "minRelChi2Change": "0.9", + "pTBinsD0ToPiK": { + "values": [ + "0.", + "5.", + "1000." + ] + }, + "pTBinsDPlusToPiKPi": { + "values": [ + "1.", + "5.", + "1000." + ] + }, + "pTBinsDsToPiKK": { + "values": [ + "1.5", + "5.", + "1000." + ] + }, + "pTBinsJpsiToEE": { + "values": [ + "0.", + "5.", + "1000." + ] + }, + "pTBinsJpsiToMuMu": { + "values": [ + "0.", + "5.", + "1000." + ] + }, + "pTBinsLcToPKPi": { + "values": [ + "4.", + "5.", + "1000." + ] + }, + "pTBinsXicToPKPi": { + "values": [ + "4.", + "5.", + "1000." + ] + }, + "propToDCA": "true" + }, + "lambdakzero-builder": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "createV0CovMats": "-1", + "dQAK0ShortMassWindow": "0.005", + "dQALambdaMassWindow": "0.005", + "dQAMaxPt": "5", + "dQANBinsMass": "400", + "dQANBinsPtCoarse": "10", + "dQANBinsRadius": "500", + "d_UseAbsDCA": "1", + "d_UseAutodetectMode": "0", + "d_UseWeightedPCA": "0", + "d_bz": "-999", + "d_doQA": "0", + "d_doTrackQA": "0", + "dcanegtopv": "0.1", + "dcapostopv": "0.1", + "dcav0dau": "1", + "geoPath": "GLO/Config/GeometryAligned", + "grpPath": "GLO/GRP/GRP", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "processRun2": "0", + "processRun3": "1", + "rejDiffCollTracks": "0", + "tpcrefit": "0", + "useMatCorrType": "2", + "v0cospa": "0.995", + "v0radius": "1" + }, + "lambdakzero-preselector": { + "dIfMCgenerateAntiHypertriton": "0", + "dIfMCgenerateAntiLambda": "0", + "dIfMCgenerateGamma": "0", + "dIfMCgenerateHypertriton": "0", + "dIfMCgenerateK0Short": "1", + "dIfMCgenerateLambda": "0", + "dPreselectOnlyBaryons": "0", + "dTPCNCrossedRows": "50", + "ddEdxPreSelectAntiHypertriton": "0", + "ddEdxPreSelectAntiLambda": "0", + "ddEdxPreSelectGamma": "0", + "ddEdxPreSelectHypertriton": "0", + "ddEdxPreSelectK0Short": "1", + "ddEdxPreSelectLambda": "0", + "ddEdxPreSelectionWindow": "7", + "processBuildAll": "1", + "processBuildMCAssociated": "0", + "processBuildValiddEdx": "0", + "processBuildValiddEdxMCAssociated": "0" + }, + "multiplicity-table": { + "processRun2": "false", + "processRun3": "true" + }, + "particles-to-tracks": { + "processIndexing": "true" + }, + "perf-k0s-resolution": { + "etaBins": { + "values": [ + 10, + -1, + 1 + ] + }, + "eventSelection": "1", + "mBins": { + "values": [ + 300, + 0.35, + 0.65 + ] + }, + "nSigTPC": "10", + "pTBins": { + "values": [ + 200, + 0, + 10 + ] + }, + "phiBins": { + "values": [ + 18, + 0, + 6.2831853 + ] + }, + "rapidity": "0.5", + "v0lifetime": "3", + "v0setting_cospa": "0.995", + "v0setting_dcanegtopv": "0.1", + "v0setting_dcapostopv": "0.1", + "v0setting_dcav0dau": "1", + "v0setting_radius": "0.9" + }, + "pseudorapidity-density": { + "estimatorEta": "1", + "exclusionPhi": { + "values": [ + [ + "4", + "5" + ] + ] + }, + "maxDCAXY": "2.4", + "maxDCAZ": "3.2", + "processGen": "true", + "processTagging": "true", + "processTrackEfficiency": "true", + "useDCAXY": "false", + "useDCAZ": "false", + "useEvSel": "false", + "usePhiCut": "false", + "usePtDCAXY": "false" + }, + "qa-efficiency": { + "applyEvSel": "2", + "do-al": "true", + "do-de": "true", + "do-el": "true", + "do-he": "true", + "do-ka": "true", + "do-mu": "true", + "do-pi": "true", + "do-pr": "true", + "do-tr": "true", + "doNegativePDG": "true", + "doPositivePDG": "true", + "doPtEta": "false", + "etaBins": { + "values": [ + "200", + "-0.8", + "0.8" + ] + }, + "globalTrackSelection": "0", + "log-pt": "0", + "make-eff": "true", + "maxProdRadius": "9999", + "nMinNumberOfContributors": "2", + "noFakesHits": "false", + "phiBins": { + "values": [ + "200", + "0", + "6.2839999198913574" + ] + }, + "processData": "true", + "processMC": "true", + "processMCWithoutCollisions": "false", + "ptBins": { + "values": [ + "200", + "0", + "5" + ] + }, + "trackSelection": "true", + "vertex-z-max": "10", + "vertex-z-min": "-10", + "yBins": { + "values": [ + "200", + "-0.5", + "0.5" + ] + } + }, + "qa-event-track": { + "binsPt": { + "values": [ + "0", + "0", + "0.10000000000000001", + "0.20000000000000001", + "0.29999999999999999", + "0.40000000000000002", + "0.5", + "0.59999999999999998", + "0.69999999999999996", + "0.80000000000000004", + "0.90000000000000002", + "1", + "1.1000000000000001", + "1.2", + "1.3", + "1.3999999999999999", + "1.5", + "2", + "5", + "10", + "20", + "50" + ] + }, + "binsTrackMultiplcity": { + "values": [ + "200", + "0", + "200" + ] + }, + "binsVertexPosXY": { + "values": [ + "500", + "-1", + "1" + ] + }, + "binsVertexPosZ": { + "values": [ + "100", + "-20", + "20" + ] + }, + "fractionOfSampledEvents": "1", + "isRun3": "true", + "maxEta": "2", + "maxPhi": "10", + "maxPt": "1e+10", + "minEta": "-2", + "minPhi": "-1", + "minPt": "-10", + "processData": "true", + "processDataIU": "false", + "processDataIUFiltered": "false", + "processMC": "true", + "processTableData": "false", + "processTableMC": "false", + "selectCharge": "0", + "selectGoodEvents": "true", + "selectMaxVtxZ": "100", + "selectPID": "0", + "selectPrim": "false", + "selectSec": "false", + "targetNumberOfEvents": "10000000", + "trackSelection": "1" + }, + "table-maker-m-c": { + "cfgBarrelLowPt": "1", + "cfgBarrelTrackCuts": "jpsiO2MCdebugCuts,kaonPID", + "cfgDetailedQA": "true", + "cfgEventCuts": "eventStandardNoINT7", + "cfgIsRun2": "false", + "cfgMCsignals": "anyBeautyHadron,everythingFromBeauty,everythingFromEverythingFromBeauty", + "cfgMuonCuts": "muonQualityCuts,muonTightQualityCutsForTests", + "cfgMuonLowPt": "1", + "cfgNoQA": "false", + "processBarrelOnly": "false", + "processBarrelOnlyWithCent": "false", + "processBarrelOnlyWithCov": "true", + "processFull": "false", + "processMuonOnlyWithCent": "false", + "processMuonOnlyWithCov": "false", + "processOnlyBCs": "true" + }, + "tof-signal": "", + "tpc-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "dEdxBins": { + "values": [ + 3000, + 0, + 1000 + ] + }, + "applyRapidityCut": "0", + "processFullWithTOFDeuteron": "0", + "phiBins": { + "values": [ + 50, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "1", + "processKaon": "0", + "processAlpha": "0", + "processFullWithTOFPion": "0", + "processFullHelium3": "1", + "minP": "0.01", + "enableDeDxPlot": "1", + "processFullWithTOFElectron": "0", + "logAxis": "1", + "processFullWithTOFAlpha": "0", + "processFullKaon": "1", + "processMuon": "0", + "processFullDeuteron": "1", + "processFullWithTOFMuon": "0", + "processFullWithTOFProton": "0", + "maxP": "5", + "processFullAlpha": "0", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "processFullWithTOFKaon": "0", + "minTPCNcls": "0", + "processFullMuon": "1", + "processFullWithTOFTriton": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullWithTOFHelium3": "0", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 100, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tof-pid-qa": { + "etaBins": { + "values": [ + 50, + -1, + 1 + ] + }, + "applyRapidityCut": "1", + "phiBins": { + "values": [ + 25, + 0, + 6.283185307179586 + ] + }, + "processProton": "0", + "processHelium3": "0", + "processFullElectron": "0", + "processKaon": "0", + "processAlpha": "0", + "processFullHelium3": "1", + "minP": "0.1", + "logAxis": "0", + "processFullKaon": "1", + "produceDeltaTEtaPhiMap": "0", + "processMuon": "0", + "processFullDeuteron": "1", + "maxP": "5", + "processFullAlpha": "0", + "ptDeltaTEtaPhiMap": "3", + "processElectron": "0", + "processDeuteron": "0", + "trackLengthBins": { + "values": [ + 50, + 0, + 1000 + ] + }, + "processFullProton": "1", + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", + "processFullMuon": "0", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 + ] + }, + "trackSelection": "1", + "processFullTriton": "1", + "processTriton": "0", + "processPion": "0", + "splitSignalPerCharge": "1", + "expSigmaBins": { + "values": [ + 200, + 0, + 200 + ] + }, + "applyEvSel": "2", + "nBinsP": "400", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "processFullPion": "1" + }, + "tpc-pid-full": { + "autofetchNetworks": "1", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "1", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "0" + }, + "pid-multiplicity": { + "processIU": "0", + "processStandard": "1" + }, + "track-extension": { + "processRun2": "false", + "processRun3": "true" + }, + "track-propagation": { + "ccdb-url": "http://alice-ccdb.cern.ch", + "geoPath": "GLO/Config/GeometryAligned", + "grpmagPath": "GLO/Config/GRPMagField", + "lutPath": "GLO/Param/MatLUT", + "mVtxPath": "GLO/Calib/MeanVertex", + "processCovariance": "true", + "processStandard": "false" + }, + "track-selection": { + "compatibilityIU": "false", + "etaMax": "0.8", + "etaMin": "-0.8", + "isRun3": "true", + "itsMatching": "1", + "produceFBextendedTable": "-1", + "ptMax": "1e+10", + "ptMin": "0.1" + }, + "emcal-correction-task": { + "nonlinearityFunction": "MC_TestbeamFinal", + "logWeight": "4.5", + "exoticCellMinAmplitude": "4", + "clusterDefinition": "kV3Default", + "useWeightExotic": "0", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellInCrossMinAmplitude": "0.1", + "maxMatchingDistance": "0.4", + "exoticCellFraction": "0.97", + "hasShaperCorrection": "1", + "isMC": "1", + "hasPropagatedTracks": "0", + "processFull": "0", + "selectedCellType": "1", + "processStandalone": "0", + "processMCFull": "1" + }, + "cell-monitor": { + "minCellTimeMain": "-50", + "maxCellTimeMain": "100", + "minCellAmplitudeTimeHists": "0.3", + "vetoBCID": "", + "selectBCID": "all", + "minCellAmplitude": "0" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "processCollisions": "0", + "doEventSel": "0", + "processAmbiguous": "1", + "clusterDefinition": "10", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "vetoBCID": "", + "selectBCID": "all", + "vertexCut": "-1" + }, + "EMCClusterMonitorTask": { + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "processCollisions": "1", + "doEventSel": "0", + "processAmbiguous": "0", + "clusterDefinition": "10", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "vetoBCID": "", + "selectBCID": "all", + "vertexCut": "-1" + }, + "emc-tmmonitor": { + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "maxTime": "20", + "clusterDefinition": "10", + "minM02": "0.1", + "vetoBCID": "", + "processCollisions": "1", + "hasPropagatedTracks": "1", + "usePionRejection": "0", + "minTime": "-25", + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "selectBCID": "all", + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "vertexCut": "-1", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + } + } +} From 602e03c32de29182e4a3f7889dd8b41102824b44 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Thu, 5 Oct 2023 15:07:18 +0200 Subject: [PATCH 0109/1239] [AnalysisQC] Add README.md --- MC/analysis_testing/README.md | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 MC/analysis_testing/README.md diff --git a/MC/analysis_testing/README.md b/MC/analysis_testing/README.md new file mode 100644 index 000000000..7da880c7c --- /dev/null +++ b/MC/analysis_testing/README.md @@ -0,0 +1,82 @@ +# Analysis testing (aka AnalysisQC) + +A collection of various analyses is managed here. This is meant for testing and it is **not** meant to replace production analysis procedures. + +Technically, it is a small set of tools and configurations. + +## Definition of analyses + +Analyses are defined in a global [configuration](../config/analysis_testing/json/analyses_config.json). Here is an example +```json +{ + "name": "EventTrackQA", + "enabled": true, + "expected_output": ["AnalysisResults.root"], + "valid_mc": true, + "valid_data": true, + "tasks": ["o2-analysis-timestamp", + "o2-analysis-track-propagation", + "o2-analysis-trackselection", + "o2-analysis-event-selection", + "o2-analysis-qa-event-track"] +} +``` +Most importantly, the tasks from `O2Physics` need to be put in a list called `tasks`. This will be translated into the common command-line pipeline. +One can specify whether a given analysis can be run on data or mc by setting `valid_mc` or `valid_data`. +To include your analysis automatically, `enabled` needs to be set to `true`. +Of course, it is important to give an analysis a short but meaningful `name`. +In order to be able to do some potentially automatic post-processing, the expected output should be specified in the list `expected_output`. + +### Analysis JSON configuration + +If no specific configuration for your analysis is found, the [defaults](../config/analysis_testing/json/default/) will be used. +It is advised however, that you add specific configurations for your analysis to not interfere with other configurations which might differ from your needs. +They must be placed at in a sub-directory that matches **exactly** the name of your analysis, see [this](../config/analysis_testing/json/EventSelectionQA/) for an example. +Each of these directories have again a sub-directory that indicates the collision system. Inside, the files **must** have the name `analysis-testing-mc.json` or `analysis-testing-data.json`. +**Note** that, whenever no specific configuration can be found, the default is taken according to the collision system and whether it is data or MC. + +## Testing an analysis on some AOD + +First, define the workflow to run analyses. This is done with +```bash +${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f [--is-mc] [-a ] [--include-disabled] +``` +To see all options, run +```bash +${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py --help +``` +By default, the tool assumes that you will be running on data. Hence, if you are interested in running on MC, you need to add the flag `--is-mc`. +The results of your analysis will be put into `/`. The default will be `Analysis/`. + +**Note** that if an analysis is disabled (`"enabled": false`, see [above](#definition-of-analyses)), it will not be included. To include it anyway, add the flag `--include-disabled`. + +By default, the workflow will be written to `workflow_analysis_test.json` which will be assumed in the following. This can be changed with `-o `. + +No, you are ready to run your analysis with +```bash +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow_analysis_test.json -tt Analysis_ +``` +The option `-tt` specifies a specific target task that should be executed. If you do not specify it, all tasks in the workflow will be executed. + +## AnalysisQC + +What is called "AnalysisQC" is build upon the shoulders of this tool set. Basically, it boils down to the usage in MC GRID productions and data reconstruction. +If a certain analysis should be executed during that procedure, the only thing that needs to be done is to add the analysis definition as explained [above](#definition-of-analyses). The `enabled` flag must be set to `true`; only those analyses are considered. +Since the defined analyses will be executed automatically, there might need to be a discussion about runtime and resource needs before corresponding requests/PRs can be taken into account. +The AnalysisQC should not introduce considerable overhead with respect to an MC production or data reconstruction themselves. + +## Further options and possibilities + +### Check if an analysis was successful + +Once you ran the analysis workflow as explained [above](#testing-an-analysis-on-some-aod), you can check if it was technically successful (that does not include any checks of the physics output). To do so, run +```bash +${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_config.py validate-output --tasks [ [...]] [-d ] +``` +It will check if the analysis went through and also if the expected outputs are there. The output directory is usually `Analysis` but you may have given another one earlier which you can set here with `-d `. + +There are more sub-commands and options that can be checked with +```bash +${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_config.py --help # OR +${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_config.py --help +``` From 0b3c0b9d97ac57f727d24c74287aabd3219b9c92 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 5 Oct 2023 17:43:16 +0200 Subject: [PATCH 0110/1239] Disable TRD calibrations in async by default --- DATA/production/configurations/asyncReco/setenv_extra.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index d06574edf..78c1b1a4c 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -450,6 +450,8 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_PHS_RUNBYRUNCALIB=0 export CALIB_PHS_L1PHASE=0 export CALIB_TRD_VDRIFTEXB=0 + export CALIB_TRD_T0=0 + export CALIB_TRD_GAIN=0 export CALIB_TPC_TIMEGAIN=0 export CALIB_TPC_RESPADGAIN=0 export CALIB_TPC_VDRIFTTGL=0 @@ -480,6 +482,9 @@ if [[ $ADD_CALIB == "1" ]]; then export DELAYINTFS_MEANVTX=55000 # 10 minutes export SVERTEXING_SOURCES=none # disable secondary vertexing fi + if [[ $ALIEN_JDL_DOTRDGAINCALIB == 1 ]]; then + export CALIB_TRD_GAIN=1 + fi if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then export CCDB_POPULATOR_UPLOAD_PATH="file://$PWD" fi From 0adf89a667ab44d8763b3f3ac0d7ce029a798e94 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 5 Oct 2023 17:46:04 +0200 Subject: [PATCH 0111/1239] Change settings for TPC residuals extraction in async --- DATA/production/configurations/asyncReco/setenv_extra.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 78c1b1a4c..516c56a5e 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -462,7 +462,8 @@ if [[ $ADD_CALIB == "1" ]]; then if [[ $DO_TPC_RESIDUAL_EXTRACTION == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 - export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds --enable-itsonly" + export CONFIG_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="scdcalib.maxTracksPerCalibSlot=35000000;scdcalib.minPtNoOuterPoint=0.2;scdcalib.maxQ2Pt=5;scdcalib.minITSNClsNoOuterPoint=5;scdcalib.minITSNCls=4;scdcalib.minTPCNClsNoOuterPoint=90" + export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds --enable-itsonly --tracking-sources ITS,TPC,ITS-TPC" # ad-hoc settings for TPC residual extraction export ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator="$ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator --output-type trackParams,unbinnedResid" if [[ $ALIEN_JDL_DEBUGRESIDUALEXTRACTION == "1" ]]; then From 114105608e15eb5fa5f32fa90c4ab7453582ff15 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 6 Oct 2023 10:23:38 +0200 Subject: [PATCH 0112/1239] [Sim WF] Adapt strangeness tracker option in AOD https://github.com/AliceO2Group/AliceO2/pull/12027 --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 3753c4ad9..f78dc40a1 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1358,7 +1358,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): AODtask['cmd'] += ' --enable-truncation 0' # developer option to suppress precision truncation if not args.with_strangeness_tracking: - AODtask['cmd'] += ' --disable-strangeness-tracking' + AODtask['cmd'] += ' --disable-strangeness-tracker' workflow['stages'].append(AODtask) From e3c36aa09286728fe683d4eed6730392579369c7 Mon Sep 17 00:00:00 2001 From: noferini Date: Fri, 6 Oct 2023 09:07:18 +0200 Subject: [PATCH 0113/1239] extend TOF time tolerance in cpass0 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 516c56a5e..51443d225 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -408,6 +408,10 @@ has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EX export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" +if [[ $ALIEN_JDL_LPMPASSNAME == "cpass0" ]]; then + CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";MatchTOF.nsigmaTimeCut=6;" +fi + # ad-hoc settings for TRD matching export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;GPU_rec_trd.minTrackPt=0.3;" From c4e463e40832d66a977dd3d283afb8e3133165e5 Mon Sep 17 00:00:00 2001 From: Luca Barioglio Date: Fri, 6 Oct 2023 09:44:20 +0200 Subject: [PATCH 0114/1239] Add A=4 hypernuclei to generator --- MC/config/PWGLF/pythia8/generator/hypernuclei.gun | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun index 09e6ef26e..55af09ba2 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun @@ -4,3 +4,5 @@ 1000020030 1 0.2 6 -1 1 1000020040 1 0.2 6 -1 1 1010010030 1 0.2 6 -1 1 +1010010040 1 0.2 6 -1 1 +1010020040 1 0.2 6 -1 1 From 1fce67fa61c4e29a972366790f5f090626b726b8 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 5 Oct 2023 22:36:39 +0200 Subject: [PATCH 0115/1239] Enable TRD gain calib ROOT output --- DATA/production/configurations/asyncReco/setenv_extra.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 51443d225..3c33219f1 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -488,6 +488,7 @@ if [[ $ADD_CALIB == "1" ]]; then export SVERTEXING_SOURCES=none # disable secondary vertexing fi if [[ $ALIEN_JDL_DOTRDGAINCALIB == 1 ]]; then + export ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow="$ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow --enable-root-output" export CALIB_TRD_GAIN=1 fi if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then From 19929493085357d68d3ebbbeb523e60a3fa88c7f Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Sun, 8 Oct 2023 14:15:28 +0200 Subject: [PATCH 0116/1239] [AnalysisQC] Align checker * at the same time, move code related to config and utility to separate file --- .../o2dpg_analysis_test_config.py | 10 +-- .../o2dpg_analysis_test_utils.py | 69 +++++++++++++++++++ .../o2dpg_analysis_test_workflow.py | 62 ++--------------- 3 files changed, 80 insertions(+), 61 deletions(-) create mode 100755 MC/analysis_testing/o2dpg_analysis_test_utils.py diff --git a/MC/analysis_testing/o2dpg_analysis_test_config.py b/MC/analysis_testing/o2dpg_analysis_test_config.py index 34bda6d19..7e9c23d83 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_config.py +++ b/MC/analysis_testing/o2dpg_analysis_test_config.py @@ -51,10 +51,6 @@ def print_status(enabled): return print("DISABLED") - def print_applicable_to(valid_keys): - for vk in valid_keys: - print(vk) - analyses = None with open (args.config, "r") as f: analyses = json.load(f)["analyses"] @@ -64,7 +60,11 @@ def print_applicable_to(valid_keys): if args.status: print_status(ana["enabled"]) if args.applicable_to: - print_applicable_to([k for k, v in ana["config"].items() if v]) + if ana.get("valid_mc", False): + print("mc") + if ana.get("valid_data", False): + print("data") + return 0 # analysis not found diff --git a/MC/analysis_testing/o2dpg_analysis_test_utils.py b/MC/analysis_testing/o2dpg_analysis_test_utils.py new file mode 100755 index 000000000..9e8fb52e3 --- /dev/null +++ b/MC/analysis_testing/o2dpg_analysis_test_utils.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +# +# Analsysis task utilities +# +import sys +from os import environ +from os.path import join, exists, abspath, expanduser + +# make sure O2DPG + O2 is loaded +O2DPG_ROOT=environ.get('O2DPG_ROOT') + +if O2DPG_ROOT is None: + print('ERROR: This needs O2DPG loaded') + sys.exit(1) + + +# some commong definitions +ANALYSIS_LABEL = "Analysis" +ANALYSIS_LABEL_ON_MC = f"{ANALYSIS_LABEL}MC" +ANALYSIS_VALID_MC = "mc" +ANALYSIS_VALID_DATA = "data" +ANALYSIS_COLLISION_SYSTEM_PP = "pp" +ANALYSIS_COLLISION_SYSTEM_PBPB = "pbpb" +ANALYSIS_CONFIGURATION_PREFIX = "analysis-testing" +ANALYSIS_DEFAULT_CONFIGURATION = {ANALYSIS_COLLISION_SYSTEM_PP: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), + ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}, + ANALYSIS_COLLISION_SYSTEM_PBPB: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), + ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}} + + +def sanitize_configuration_path(path): + # sanitize path + path = path.replace("json://", "") + if path[0] != "$": + # only do this if there is no potential environment variable given as the first part of the path + path = abspath(expanduser(path)) + return f"json://{path}" + + +def get_default_configuration(data_or_mc, collision_system): + path = ANALYSIS_DEFAULT_CONFIGURATION.get(collision_system, None) + if not path: + print(f"ERROR: Unknown collision system {collision_system}") + return None + return path[data_or_mc] + + +def get_configuration(analysis_name, data_or_mc, collision_system): + path = join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", analysis_name, collision_system, f"{ANALYSIS_CONFIGURATION_PREFIX}-{data_or_mc}.json") + if not exists(path): + path = get_default_configuration(data_or_mc, collision_system) + if not path: + return None + print(f"INFO: Use default configuration for {analysis_name}") + return sanitize_configuration_path(path) + + return sanitize_configuration_path(path) + + +def get_collision_system(collision_system=None): + if not collision_system: + return environ.get("ALIEN_JDL_LPMINTERACTIONTYPE", "pp").lower() + return collision_system.lower() + + +def full_ana_name(raw_ana_name): + """Make the standard name of the analysis how it should appear in the workflow""" + return f"{ANALYSIS_LABEL}_{raw_ana_name}" diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 2d6f11cf8..ad90ecbe1 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -82,62 +82,12 @@ spec.loader.exec_module(o2dpg_workflow_utils) from o2dpg_workflow_utils import createTask, dump_workflow -####################### -# ANALYSIS definition # -####################### - -# some commong definitions -ANALYSIS_LABEL = "Analysis" -ANALYSIS_LABEL_ON_MC = f"{ANALYSIS_LABEL}MC" -ANALYSIS_VALID_MC = "mc" -ANALYSIS_VALID_DATA = "data" -ANALYSIS_COLLISION_SYSTEM_PP = "pp" -ANALYSIS_COLLISION_SYSTEM_PBPB = "pbpb" -ANALYSIS_CONFIGURATION_PREFIX = "analysis-testing" -ANALYSIS_DEFAULT_CONFIGURATION = {ANALYSIS_COLLISION_SYSTEM_PP: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), - ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PP, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}, - ANALYSIS_COLLISION_SYSTEM_PBPB: {ANALYSIS_VALID_MC: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_MC}.json"), - ANALYSIS_VALID_DATA: join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", "default", ANALYSIS_COLLISION_SYSTEM_PBPB, f"{ANALYSIS_CONFIGURATION_PREFIX}-{ANALYSIS_VALID_DATA}.json")}} - - -def sanitize_configuration_path(path): - # sanitize path - path = path.replace("json://", "") - if path[0] != "$": - # only do this if there is no potential environment variable given as the first part of the path - path = abspath(expanduser(path)) - return f"json://{path}" - - -def get_default_configuration(data_or_mc, collision_system): - path = ANALYSIS_DEFAULT_CONFIGURATION.get(collision_system, None) - if not path: - print(f"ERROR: Unknown collision system {collision_system}") - return None - return path[data_or_mc] - - -def get_configuration(analysis_name, data_or_mc, collision_system): - path = join(O2DPG_ROOT, "MC", "config", "analysis_testing", "json", analysis_name, collision_system, f"{ANALYSIS_CONFIGURATION_PREFIX}-{data_or_mc}.json") - if not exists(path): - path = get_default_configuration(data_or_mc, collision_system) - if not path: - return None - print(f"INFO: Use default configuration for {analysis_name}") - return sanitize_configuration_path(path) - - return sanitize_configuration_path(path) - - -def get_collision_system(collision_system=None): - if not collision_system: - return environ.get("ALIEN_JDL_LPMINTERACTIONTYPE", "pp").lower() - return collision_system.lower() - - -def full_ana_name(raw_ana_name): - """Make the standard name of the analysis how it should appear in the workflow""" - return f"{ANALYSIS_LABEL}_{raw_ana_name}" +module_name = "o2dpg_analysis_test_utils" +spec = importlib.util.spec_from_file_location(module_name, join(O2DPG_ROOT, "MC", "analysis_testing", "o2dpg_analysis_test_utils.py")) +o2dpg_analysis_test_utils = importlib.util.module_from_spec(spec) +sys.modules[module_name] = o2dpg_analysis_test_utils +spec.loader.exec_module(o2dpg_analysis_test_utils) +from o2dpg_analysis_test_utils import * def create_ana_task(name, cmd, output_dir, *, needs=None, shmsegmentsize="--shm-segment-size 2000000000", From 9f29e5f75bec27d5743e8703253cb5a0768a0066 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 9 Oct 2023 13:29:41 +0200 Subject: [PATCH 0117/1239] PbPb needs different settings --- .../configurations/asyncReco/async_pass.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 15e8c867b..3a36d31d7 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -385,13 +385,19 @@ else if [[ "0$ASYNC_PASS_NO_OPTIMIZED_DEFAULTS" != "01" ]]; then if [[ "$ALIEN_JDL_EPNFULLNUMACPUONLY" != 1 ]]; then export TIMEFRAME_RATE_LIMIT=3 - if (( $(echo "$RUN_IR > 800000" | bc -l) )); then - export TIMEFRAME_RATE_LIMIT=1 - elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then - export TIMEFRAME_RATE_LIMIT=6 + if [[ $BEAMTYPE == "pp" ]]; then + if (( $(echo "$RUN_IR > 800000" | bc -l) )); then + export TIMEFRAME_RATE_LIMIT=1 + elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then + export TIMEFRAME_RATE_LIMIT=6 + fi + export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) + export SHMSIZE=16000000000 + else # PbPb + export TIMEFRAME_RATE_LIMIT=3 + export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu + export SHMSIZE=16000000000 fi - export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) - export SHMSIZE=16000000000 else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs export TIMEFRAME_RATE_LIMIT=32 From 483f824f3c8357adf2fa2dbddb12f8e6d799cf17 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 9 Oct 2023 13:49:34 +0200 Subject: [PATCH 0118/1239] Updating json for MC --- MC/bin/o2dpg_sim_workflow.py | 2 +- MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index f78dc40a1..aa3fb5c26 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1249,7 +1249,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/vertexing-qc-direct-mc.json') addQCPerTF(taskName='ITSTPCmatchQC', needs=[ITSTPCMATCHtask['name']], - readerCommand='o2-global-track-cluster-reader --track-types "TPC,ITS-TPC"', + readerCommand='o2-global-track-cluster-reader --track-types "ITS,TPC,ITS-TPC" ', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json') if isActive('TOF'): addQCPerTF(taskName='TOFMatchQC', diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index 93a42663f..9f6f9eae6 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -41,7 +41,7 @@ "dataSource" : { "type" : "direct", "query_comment" : "checking every matched track", - "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC" + "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackITS:ITS/TRACKS/0;trackITSROF:ITS/ITSTrackROF/0;trackITSClIdx:ITS/TRACKCLSID/0;alpparITS:ITS/ALPIDEPARAM/0?lifetime=condition&ccdb-path=ITS/Config/AlpideParam;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;trackITSMCTR:ITS/TRACKSMCTR" }, "taskParameters" : { "GID" : "ITS-TPC,ITS", From 032bedd7f49a9f3815b7e20f7d3cc9f3b1dea888 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Mon, 9 Oct 2023 16:22:51 +0200 Subject: [PATCH 0119/1239] [EMCAL-756] Update async QC configuration for PbPb for EMCAL --- DATA/production/qc-async/emc_PbPb.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index a91957c10..e3e64c10f 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -69,9 +69,11 @@ "clusterizerMaxTime": "300.", "clusterizerMaxTimeDelta": "1000.", "hasInvMassMesons": "true", + "mesonClusterMinE": "1.5", "mesonClustersRejectExotics": 1, "mesonClusterMaxTime": "50.", - "MultiplicityRange":"400" + "MultiplicityRange":"400", + "mesonMinPt": "7." } }, "BCTaskEMCAL": { From b4bba9b58471450e7f305e584579db7e7fc6aa24 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Mon, 9 Oct 2023 16:44:53 +0200 Subject: [PATCH 0120/1239] [EMCAL-756] Add pileup cut for total energy histograms Total energy has pileup tail, 25 ns cut sufficient for a 50 ns filling scheme. --- DATA/production/qc-async/emc_PbPb.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index e3e64c10f..8706a1f89 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -42,7 +42,8 @@ "MultiplicityRangeSMThreshold": "200", "TotalEnergyRangeDetector": "700", "TotalEnergyRangeSM": "200", - "TotalEnergyRange": "2000" + "TotalEnergyRange": "2000", + "TotalEnergyMaxCellTime": "25" } }, "ClusterTaskEMCAL": { From 1f19859a8d7531a9f4e0b4e096a889c2af7fad98 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 10 Oct 2023 15:17:15 +0200 Subject: [PATCH 0121/1239] Updating PbPb settings --- DATA/production/configurations/asyncReco/async_pass.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 3a36d31d7..fce992d80 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -394,7 +394,7 @@ else export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) export SHMSIZE=16000000000 else # PbPb - export TIMEFRAME_RATE_LIMIT=3 + export TIMEFRAME_RATE_LIMIT=2 export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu export SHMSIZE=16000000000 fi From 75ada2504ad63cc41f4d66cad67073bd2b2fc05e Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 12 Oct 2023 11:08:49 +0200 Subject: [PATCH 0122/1239] Update TPC residuals extraction settings for async again --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 3c33219f1..bc7984834 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -466,8 +466,8 @@ if [[ $ADD_CALIB == "1" ]]; then if [[ $DO_TPC_RESIDUAL_EXTRACTION == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 - export CONFIG_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="scdcalib.maxTracksPerCalibSlot=35000000;scdcalib.minPtNoOuterPoint=0.2;scdcalib.maxQ2Pt=5;scdcalib.minITSNClsNoOuterPoint=5;scdcalib.minITSNCls=4;scdcalib.minTPCNClsNoOuterPoint=90" - export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds --enable-itsonly --tracking-sources ITS,TPC,ITS-TPC" + export CONFIG_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="scdcalib.maxTracksPerCalibSlot=35000000;scdcalib.minPtNoOuterPoint=0.2;scdcalib.maxQ2Pt=5;scdcalib.minITSNClsNoOuterPoint=6;scdcalib.minITSNCls=4;scdcalib.minTPCNClsNoOuterPoint=90" + export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds --enable-itsonly" # ad-hoc settings for TPC residual extraction export ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator="$ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator --output-type trackParams,unbinnedResid" if [[ $ALIEN_JDL_DEBUGRESIDUALEXTRACTION == "1" ]]; then From d2f17c9d51057bf862d103379bc857c1277cae10 Mon Sep 17 00:00:00 2001 From: shahoian Date: Thu, 12 Oct 2023 16:05:00 +0200 Subject: [PATCH 0123/1239] Use optional TPC_CORR_SCALING env.var to tpc-laser_filter --- DATA/production/calib/tpc-laser-filter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 3f26160e9..bc3ce3792 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -64,7 +64,7 @@ o2-dpl-raw-proxy $ARGS_ALL \ --pipeline tpc-raw-to-digits-0:20 \ --remove-duplicates \ --send-ce-digits \ - | o2-tpc-reco-workflow $ARGS_ALL \ + | o2-tpc-reco-workflow $ARGS_ALL ${TPC_CORR_SCALING:-} \ --input-type digitizer \ --output-type "tracks,disable-writer" \ --disable-mc \ From 3a29e95794817c5c1e2a5af21c967b9e1710bffe Mon Sep 17 00:00:00 2001 From: Matthias Kleiner Date: Wed, 11 Oct 2023 15:20:20 +0200 Subject: [PATCH 0124/1239] Adding workflow for TPC and ITS time series --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index bc7984834..a122a5edd 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -508,6 +508,12 @@ if [[ $ALIEN_JDL_EXTRACTCURRENTS == 1 ]]; then has_detector_reco TPC && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-integrate-cluster-workflow" fi +# extra workflows in case we want to process the currents for time series +if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then + if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi + has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-time-series-workflow" +fi + # Enabling AOD if [[ $ALIEN_JDL_AODOFF != "1" ]]; then export WORKFLOW_PARAMETERS="AOD,${WORKFLOW_PARAMETERS}" From 90f420716a82c21eaae1449e91059825a77251af Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 13 Oct 2023 15:44:51 +0200 Subject: [PATCH 0125/1239] Possibility to disable strangeness tracking and cascades from JDL --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index a122a5edd..b5bd7481c 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -388,6 +388,12 @@ fi # secondary vertexing +if [[ $ALIEN_JDL_DISABLESTRTRACKING == 1 ]]; then + export STRTRACKING=" --disable-strangeness-tracker " +fi +if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then + export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " +fi export SVTX="svertexer.checkV0Hypothesis=false;svertexer.checkCascadeHypothesis=false" # strangeness tracking export STRK="" From 32be41c3e123b41ecf98148d987f846df2f854a6 Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Thu, 12 Oct 2023 15:53:25 +0200 Subject: [PATCH 0126/1239] ITS-vertexer: disable low mult seeds for async PbPb --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index b5bd7481c..f7d6be56b 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -343,7 +343,7 @@ export ITSEXTRAERR="ITSCATrackerParam.sysErrY2[0]=$ERRIB;ITSCATrackerParam.sysEr # ad-hoc options for ITS reco workflow EXTRA_ITSRECO_CONFIG= if [[ $BEAMTYPE == "PbPb" ]]; then - EXTRA_ITSRECO_CONFIG="ITSCATrackerParam.trackletsPerClusterLimit=5.;ITSCATrackerParam.cellsPerClusterLimit=5.;ITSVertexerParam.clusterContributorsCut=16;" + EXTRA_ITSRECO_CONFIG="ITSCATrackerParam.trackletsPerClusterLimit=5.;ITSCATrackerParam.cellsPerClusterLimit=5.;ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" elif [[ $BEAMTYPE == "pp" ]]; then EXTRA_ITSRECO_CONFIG="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" fi From ee18344eb8b68eacbdd22ad7b6678fdd01e8b203 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 15 Oct 2023 21:24:09 +0200 Subject: [PATCH 0127/1239] Don't run topology merger post-caching command if workflowmode = print --- DATA/tools/epn/gen_topo_o2dpg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index e9aa5a03c..29346796d 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -103,7 +103,7 @@ while true; do break done -if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]]; then +if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]] && [[ "0$WORKFLOWMODE" != "0print" ]]; then TMP_POST_CACHING_CMD="$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" TMP_POST_CACHING_NMIN=$(( $RECO_NUM_NODES_OVERRIDE > $RECO_MAX_FAIL_NODES_OVERRIDE ? $RECO_NUM_NODES_OVERRIDE - $RECO_MAX_FAIL_NODES_OVERRIDE : 0 )) TMP_POST_CACHING_CMD+=" --nodes-mi50 $RECO_NUM_NODES_OVERRIDE --nmin-mi50 $TMP_POST_CACHING_NMIN" From 752049396bbbf7e931b259a59b5c26726af80c50 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Sun, 15 Oct 2023 21:43:42 +0200 Subject: [PATCH 0128/1239] Modernize TRD noise workflow (#1282) --- DATA/testing/detectors/TRD/trd-noise.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/testing/detectors/TRD/trd-noise.sh b/DATA/testing/detectors/TRD/trd-noise.sh index ea71eac67..4b84b1ef6 100755 --- a/DATA/testing/detectors/TRD/trd-noise.sh +++ b/DATA/testing/detectors/TRD/trd-noise.sh @@ -3,6 +3,7 @@ # Make common arguments and helper functions such as add_W available source common/setenv.sh source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh # Define input data required by DPL (in this case all RAWDATA from TRD) @@ -27,7 +28,7 @@ add_W o2-calibration-trd-workflow "--noise --calib-dds-collection-index 0" add_W o2-calibration-ccdb-populator-workflow "--ccdb-path $TRD_CCDB_PATH" # Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) -WORKFLOW+="o2-dpl-run $ARGS_ALL" +WORKFLOW+="o2-dpl-run $GLOBALDPLOPT $ARGS_ALL" [[ $WORKFLOWMODE != "print" ]] && WORKFLOW+=" --${WORKFLOWMODE} ${WORKFLOWMODE_FILE:-}" [[ $WORKFLOWMODE == "print" || "${PRINT_WORKFLOW:-}" == "1" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2') if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi From 8c16b0cf2e61db02032208104424707188d14951 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Fri, 13 Oct 2023 14:24:14 +0200 Subject: [PATCH 0129/1239] [AnalysisQC] Enable common args per analysis * use for instance --add-common-args EMCAL-shm-segment-size 2500000000 which will be forwarded as a common argument and appended to the analysis pipeline of EMCAL analysis as --shm-segment-size 2500000000 * Use also e.g. ALL-readers to influence set --readers for all analyses * - supersedes ALL- --- MC/analysis_testing/analysis_test.sh | 12 ++++- .../o2dpg_analysis_test_utils.py | 47 +++++++++++++++++++ .../o2dpg_analysis_test_workflow.py | 23 +++++---- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/MC/analysis_testing/analysis_test.sh b/MC/analysis_testing/analysis_test.sh index 4724216f6..e934b6d47 100755 --- a/MC/analysis_testing/analysis_test.sh +++ b/MC/analysis_testing/analysis_test.sh @@ -24,6 +24,7 @@ NTF=$(find ./ -name "tf*" -type d | wc | awk '//{print $1}') include_disabled= testanalysis= aod= +add_common_args= if [[ "${#}" == "1" ]] ; then # make it backward-compatible @@ -48,6 +49,12 @@ else shift shift ;; + --add-common-args) + add_common_args=" ${2} ${3} " + shift + shift + shift + ;; *) echo "ERROR: Unknown argument ${1}" exit 1 @@ -59,6 +66,7 @@ fi # basic checks [[ "${testanalysis}" == "" ]] && { echo "ERROR: No analysis specified to be run" ; exit 1 ; } [[ "${aod}" == "" ]] && { echo "ERROR: No AOD found to be analysed" ; exit 1 ; } +[[ "${add_common_args}" != "" ]] && add_common_args="--add-common-args ${add_common_args}" # check if enabled enabled=$($O2DPG_ROOT/MC/analysis_testing/o2dpg_analysis_test_config.py check -t ${testanalysis} --status) @@ -68,7 +76,9 @@ enabled=$($O2DPG_ROOT/MC/analysis_testing/o2dpg_analysis_test_config.py check -t mkdir Analysis 2>/dev/null include_disabled=${include_disabled:+--include-disabled} workflow_path="Analysis/workflow_analysis_test_${testanalysis}.json" -$O2DPG_ROOT/MC/analysis_testing/o2dpg_analysis_test_workflow.py --is-mc -f ${aod} -o ${workflow_path} --only-analyses ${testanalysis} ${include_disabled} +rm ${workflow_path} 2>/dev/null +$O2DPG_ROOT/MC/analysis_testing/o2dpg_analysis_test_workflow.py --is-mc -f ${aod} -o ${workflow_path} --only-analyses ${testanalysis} ${include_disabled} ${add_common_args} +[[ ! -f "${workflow_path}" ]] && { echo "Could not construct workflow for analysis ${testanalysis}" ; exit 1 ; } $O2DPG_ROOT/MC/bin/o2_dpg_workflow_runner.py -f ${workflow_path} -tt Analysis_${testanalysis}$ --rerun-from Analysis_${testanalysis}$ RC=$? diff --git a/MC/analysis_testing/o2dpg_analysis_test_utils.py b/MC/analysis_testing/o2dpg_analysis_test_utils.py index 9e8fb52e3..1a3901fe6 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_utils.py +++ b/MC/analysis_testing/o2dpg_analysis_test_utils.py @@ -67,3 +67,50 @@ def get_collision_system(collision_system=None): def full_ana_name(raw_ana_name): """Make the standard name of the analysis how it should appear in the workflow""" return f"{ANALYSIS_LABEL}_{raw_ana_name}" + + +def get_common_args_as_string(analysis_name, all_common_args): + """ + all_common_args is of the form + [-shm-segment-size , -readers , ...] + + Find common arguments for this specific analysis + """ + + def make_args_string(args_map_in): + out_string = "" + for key, value in args_map_in.items(): + out_string += f" --{key} {value}" + return out_string + + # default arguments for all analyses + args_map = {"shm-segment-size": 2000000000, + "readers": 1, + "aod-memory-rate-limit": 500000000} + + # arguments dedicated for this analysis + args_map_overwrite = {} + + if not all_common_args: + return make_args_string(args_map) + + if len(all_common_args) % 2: + print("ERROR: Cannot digest common args.") + return None + + for i in range(0, len(all_common_args), 2): + tokens = all_common_args[i].split("-") + key = "-".join(tokens[1:]) + if tokens[0] == analysis_name: + # for this analysis, add to dedicated dict + args_map_overwrite[key] = all_common_args[i+1] + continue + if tokens[0] == "ALL": + # otherwise add to default dict + args_map[key] = all_common_args[i+1] + + # overwrite default dict with dedicated arguments + for key, value in args_map_overwrite.items(): + args_map[key] = value + + return make_args_string(args_map) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index ad90ecbe1..a3a3e37c8 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -90,9 +90,7 @@ from o2dpg_analysis_test_utils import * -def create_ana_task(name, cmd, output_dir, *, needs=None, shmsegmentsize="--shm-segment-size 2000000000", - aodmemoryratelimit="--aod-memory-rate-limit 500000000", - readers="--readers 1", extraarguments="-b", is_mc=False): +def create_ana_task(name, cmd, output_dir, *, needs=None, extraarguments="-b", is_mc=False): """Quick helper to create analysis task This creates an analysis task from various arguments @@ -107,12 +105,6 @@ def create_ana_task(name, cmd, output_dir, *, needs=None, shmsegmentsize="--shm- Keyword args (optional): needs: tuple, list list of other tasks to be run before - shmsegmentsize: str - O2/DPL argument string for shared mem size - aodmemoryratelimit: str - O2/DPL argument string for AOD memory rate limit - readers: O2/DPL argument string - number of readers extraarguments: str O2/DPL argument string for any other desired arguments to be added to the executed cmd Return: @@ -125,7 +117,7 @@ def create_ana_task(name, cmd, output_dir, *, needs=None, shmsegmentsize="--shm- task = createTask(name=full_ana_name(name), cwd=join(output_dir, name), lab=[ANALYSIS_LABEL, name], cpu=1, mem='2000', needs=needs) if is_mc: task["labels"].append(ANALYSIS_LABEL_ON_MC) - task['cmd'] = f"{cmd} {shmsegmentsize} {aodmemoryratelimit} {readers} {extraarguments}" + task['cmd'] = f"{cmd} {extraarguments}" return task @@ -220,7 +212,7 @@ def get_additional_workflows(input_aod): return additional_workflows -def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis", *, analyses_only=None, is_mc=True, collision_system=None, needs=None, autoset_converters=False, include_disabled_analyses=False, timeout=None): +def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis", *, analyses_only=None, is_mc=True, collision_system=None, needs=None, autoset_converters=False, include_disabled_analyses=False, timeout=None, add_common_args=None): """Add default analyses to user workflow Args: @@ -265,12 +257,18 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis continue print(f"INFO: Analysis {ana['name']} uses configuration {configuration}") + add_common_args_ana = get_common_args_as_string(ana["name"], add_common_args) + if not add_common_args_ana: + print(f"ERROR: Cannot parse common args for analysis {ana['name']}") + continue + for i in additional_workflows: if i not in ana["tasks"]: # print("Appending extra task", i, "to analysis", ana["name"], "as it is not there yet and needed for conversion") ana["tasks"].append(i) piped_analysis = f" --configuration {configuration} | ".join(ana["tasks"]) piped_analysis += f" --configuration {configuration} --aod-file {input_aod}" + piped_analysis += add_common_args_ana if timeout is not None: piped_analysis += f" --time-limit {timeout}" workflow.append(create_ana_task(ana["name"], piped_analysis, output_dir, needs=needs, is_mc=is_mc)) @@ -330,7 +328,7 @@ def run(args): return 1 workflow = [] - add_analysis_tasks(workflow, args.input_file, expanduser(args.analysis_dir), is_mc=args.is_mc, analyses_only=args.only_analyses, autoset_converters=args.autoset_converters, include_disabled_analyses=args.include_disabled, timeout=args.timeout, collision_system=args.collision_system) + add_analysis_tasks(workflow, args.input_file, expanduser(args.analysis_dir), is_mc=args.is_mc, analyses_only=args.only_analyses, autoset_converters=args.autoset_converters, include_disabled_analyses=args.include_disabled, timeout=args.timeout, collision_system=args.collision_system, add_common_args=args.add_common_args) if args.with_qc_upload: add_analysis_qc_upload_tasks(workflow, args.period_name, args.run_number, args.pass_name) if not workflow: @@ -356,6 +354,7 @@ def main(): parser.add_argument("--autoset-converters", dest="autoset_converters", action="store_true", help="Compatibility mode to automatically set the converters for the analysis") parser.add_argument("--timeout", type=int, default=None, help="Timeout for analysis tasks in seconds.") parser.add_argument("--collision-system", dest="collision_system", help="Set the collision system. If not set, tried to be derived from ALIEN_JDL_LPMInterationType. Fallback to pp") + parser.add_argument("--add-common-args", dest="add_common_args", nargs="*", help="Pass additional common arguments per analysis, for instance --add-common-args EMCAL-shm-segment-size 2500000000 will add --shm-segment-size 2500000000 to the EMCAL analysis") parser.set_defaults(func=run) args = parser.parse_args() return(args.func(args)) From ce22f9c99e2199c3fe2e6db769ff6d533acb1955 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 17 Oct 2023 17:02:18 +0200 Subject: [PATCH 0130/1239] TRD gain calibration settings for async --- DATA/production/configurations/asyncReco/setenv_extra.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index f7d6be56b..0efa19765 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -494,6 +494,7 @@ if [[ $ADD_CALIB == "1" ]]; then export SVERTEXING_SOURCES=none # disable secondary vertexing fi if [[ $ALIEN_JDL_DOTRDGAINCALIB == 1 ]]; then + export CONFIG_EXTRA_PROCESS_o2_calibration_trd_workflow="TRDCalibParams.minEntriesChamberGainCalib=999999999;TRDCalibParams.minEntriesTotalGainCalib=10000;TRDCalibParams.nTrackletsMinGainCalib=4" export ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow="$ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow --enable-root-output" export CALIB_TRD_GAIN=1 fi From d5e8d04a6a6fac7def867e32e016dd022b94ac79 Mon Sep 17 00:00:00 2001 From: Andreas Molander Date: Wed, 4 Oct 2023 14:46:50 +0200 Subject: [PATCH 0131/1239] FV0: enable amp vs time histos for each channel in async QC --- DATA/production/qc-async/fv0.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/fv0.json b/DATA/production/qc-async/fv0.json index 133507c3f..872230f68 100644 --- a/DATA/production/qc-async/fv0.json +++ b/DATA/production/qc-async/fv0.json @@ -36,7 +36,8 @@ "query": "digits:FV0/DIGITSBC/0;channels:FV0/DIGITSCH/0" }, "taskParameters": { - "trgModeInnerOuterThresholdVar": "Ampl", + "ChannelIDsAmpVsTime": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48", + "trgModeInnerOuterThresholdVar": "Ampl", "trgThresholdNChannels": "2", "trgThresholdCharge": "8", "trgThresholdChargeInner": "4", From 956a3789226d083c217ce5728c769ead34a13813 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:57:36 +0200 Subject: [PATCH 0132/1239] Tpc laser new (#1266) * Change path to qc files for calibration files * Add fixes for Error messages in Laser workflow --- DATA/production/calib/tpc-laser-aggregator.sh | 2 +- DATA/production/calib/tpc-laser-filter.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DATA/production/calib/tpc-laser-aggregator.sh b/DATA/production/calib/tpc-laser-aggregator.sh index 2fc461a38..01f7864a1 100755 --- a/DATA/production/calib/tpc-laser-aggregator.sh +++ b/DATA/production/calib/tpc-laser-aggregator.sh @@ -4,7 +4,7 @@ source common/setenv.sh source common/getCommonArgs.sh -PROXY_INSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSREFS" +PROXY_INSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;D:TPC/CLUSREFS" CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index bc3ce3792..fb3c0cf3d 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -41,7 +41,7 @@ fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;eos:***/INFORMATION;D:TPC/CLUSREFS" +PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;D:TPC/CLUSREFS" LASER_DECODER_ADD='' @@ -71,7 +71,7 @@ o2-dpl-raw-proxy $ARGS_ALL \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ $GPU_CONFIG \ --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \ - --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35" \ + --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35;GPU_proc.memoryScalingFactor=2" \ | o2-tpc-laser-track-filter $ARGS_ALL \ | o2-dpl-output-proxy ${ARGS_ALL} \ --dataspec "$PROXY_OUTSPEC" \ From 9bd89aa554d831c63f7d6dd6dcb6fcb9baa0af03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Thu, 19 Oct 2023 10:23:52 +0200 Subject: [PATCH 0133/1239] Streamline LF injector/GT, add eta window trigger (#1247) * Streamline LF injector/GT, add eta window trigger * Add 900 GeV triggered for strangeness studies * Use const members * Add json support --- ...GeneratorLFStrangenessTriggered_900gev.ini | 10 ++ .../GeneratorLFStrangenessTriggered_900gev.C | 58 ++++++ MC/config/PWGLF/pythia8/generator/nuclei.gun | 18 +- .../PWGLF/pythia8/generator/resonancelist.gun | 40 ++--- .../pythia8/generator/strangeparticlelist.gun | 10 +- .../generator/strangeparticlelistgun.json | 38 ++++ .../PWGLF/pythia8/generator_pythia8_LF.C | 166 ++++++++++++------ .../run_StrangenessTriggered_pp900gev.sh | 44 +++++ 8 files changed, 292 insertions(+), 92 deletions(-) create mode 100644 MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_900gev.ini create mode 100644 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C create mode 100644 MC/config/PWGLF/pythia8/generator/strangeparticlelistgun.json create mode 100755 MC/run/PWGLF/run_StrangenessTriggered_pp900gev.sh diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_900gev.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_900gev.ini new file mode 100644 index 000000000..3447669ac --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_900gev.ini @@ -0,0 +1,10 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 0) +funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 4) + +[GeneratorPythia8] # if triggered then this will be used as the background event +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg + +[DecayerPythia8] +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C new file mode 100644 index 000000000..49ba5d4e2 --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C @@ -0,0 +1,58 @@ +int External() +{ + std::string path{"o2sim_Kine.root"}; + int numberOfInjectedSignalsPerEvent{1}; + std::vector injectedPDGs = { + 3334, + -3334, + 3312, + -3312}; + + auto nInjection = injectedPDGs.size(); + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + if (!tree) { + std::cerr << "Cannot find tree o2sim in file " << path << "\n"; + return 1; + } + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + std::vector nSignal; + for (int i = 0; i < nInjection; i++) { + nSignal.push_back(0); + } + + auto nEvents = tree->GetEntries(); + for (int i = 0; i < nEvents; i++) { + auto check = tree->GetEntry(i); + for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) { + auto track = tracks->at(idxMCTrack); + auto pdg = track.GetPdgCode(); + auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); + int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG + if (it != injectedPDGs.end()) // found + { + // count signal PDG + nSignal[index]++; + } + } + } + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + for (int i = 0; i < nInjection; i++) { + std::cout << "# Injected nuclei \n"; + std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; + if (nSignal[i] == 0) { + std::cerr << "No generated: " << injectedPDGs[i] << "\n"; + return 1; // At least one of the injected particles should be generated + } + } + return 0; +} \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator/nuclei.gun b/MC/config/PWGLF/pythia8/generator/nuclei.gun index e33c00a82..e1bbc7d88 100644 --- a/MC/config/PWGLF/pythia8/generator/nuclei.gun +++ b/MC/config/PWGLF/pythia8/generator/nuclei.gun @@ -1,9 +1,9 @@ -# PDG N ptMin ptMax genDecayed -1000010020 10 0.2 10 --1000010020 10 0.2 10 -1000010030 10 0.2 10 --1000010030 10 0.2 10 -1000020030 10 0.2 10 --1000020030 10 0.2 10 -1000020040 10 0.2 10 --1000020040 10 0.2 10 +# PDG N ptMin ptMax etaMin etaMax genDecayed +1000010020 10 0.2 10 -1.2 1.2 +-1000010020 10 0.2 10 -1.2 1.2 +1000010030 10 0.2 10 -1.2 1.2 +-1000010030 10 0.2 10 -1.2 1.2 +1000020030 10 0.2 10 -1.2 1.2 +-1000020030 10 0.2 10 -1.2 1.2 +1000020040 10 0.2 10 -1.2 1.2 +-1000020040 10 0.2 10 -1.2 1.2 diff --git a/MC/config/PWGLF/pythia8/generator/resonancelist.gun b/MC/config/PWGLF/pythia8/generator/resonancelist.gun index b52df8c0d..daedb88f0 100644 --- a/MC/config/PWGLF/pythia8/generator/resonancelist.gun +++ b/MC/config/PWGLF/pythia8/generator/resonancelist.gun @@ -1,20 +1,20 @@ -# PDG N ptMin ptMax DecayByGenerator -313 10 0.2 10 genDecayed --313 10 0.2 10 genDecayed -323 10 0.2 10 genDecayed --323 10 0.2 10 genDecayed -333 10 0.2 10 genDecayed -9010221 10 0.2 10 genDecayed -113 10 0.2 10 genDecayed -213 10 0.2 10 genDecayed --213 10 0.2 10 genDecayed -3114 10 0.2 10 genDecayed --3114 10 0.2 10 genDecayed -3224 10 0.2 10 genDecayed --3224 10 0.2 10 genDecayed -3124 10 0.2 10 genDecayed --3124 10 0.2 10 genDecayed -3324 10 0.2 10 genDecayed --3324 10 0.2 10 genDecayed -10323 10 0.2 10 genDecayed --10323 10 0.2 10 genDecayed \ No newline at end of file +# PDG N ptMin ptMax etaMin etaMax DecayByGenerator +313 10 0.2 10 -1.2 1.2 genDecayed +-313 10 0.2 10 -1.2 1.2 genDecayed +323 10 0.2 10 -1.2 1.2 genDecayed +-323 10 0.2 10 -1.2 1.2 genDecayed +333 10 0.2 10 -1.2 1.2 genDecayed +9010221 10 0.2 10 -1.2 1.2 genDecayed +113 10 0.2 10 -1.2 1.2 genDecayed +213 10 0.2 10 -1.2 1.2 genDecayed +-213 10 0.2 10 -1.2 1.2 genDecayed +3114 10 0.2 10 -1.2 1.2 genDecayed +-3114 10 0.2 10 -1.2 1.2 genDecayed +3224 10 0.2 10 -1.2 1.2 genDecayed +-3224 10 0.2 10 -1.2 1.2 genDecayed +3124 10 0.2 10 -1.2 1.2 genDecayed +-3124 10 0.2 10 -1.2 1.2 genDecayed +3324 10 0.2 10 -1.2 1.2 genDecayed +-3324 10 0.2 10 -1.2 1.2 genDecayed +10323 10 0.2 10 -1.2 1.2 genDecayed +-10323 10 0.2 10 -1.2 1.2 genDecayed \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun b/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun index f1e5a65d1..cf8fbc9d3 100644 --- a/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun +++ b/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun @@ -1,5 +1,5 @@ -# PDG N ptMin ptMax genDecayed -3334 1 0.2 10 --3334 1 0.2 10 -3312 1 0.2 10 --3312 1 0.2 10 \ No newline at end of file +# PDG N ptMin ptMax etaMin etaMax genDecayed +3334 1 0.2 10 -1.2 1.2 +-3334 1 0.2 10 -1.2 1.2 +3312 1 0.2 10 -1.2 1.2 +-3312 1 0.2 10 -1.2 1.2 \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator/strangeparticlelistgun.json b/MC/config/PWGLF/pythia8/generator/strangeparticlelistgun.json new file mode 100644 index 000000000..84ba4a4c9 --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/strangeparticlelistgun.json @@ -0,0 +1,38 @@ +{ + "omega-": { + "pdg": 3334, + "n": 1, + "ptMin": 0.2, + "ptMax": 10, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": false + }, + "antiomega+": { + "pdg": -3334, + "n": 1, + "ptMin": 0.2, + "ptMax": 10, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": false + }, + "xi-": { + "pdg": 3312, + "n": 1, + "ptMin": 0.2, + "ptMax": 10, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": false + }, + "antixi+": { + "pdg": -3312, + "n": 1, + "ptMin": 0.2, + "ptMax": 10, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": false + } +} \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_LF.C b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C index 4428c94de..b5550fb5e 100644 --- a/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +++ b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C @@ -23,6 +23,7 @@ #include #include "Generators/GeneratorPythia8Param.h" #include "Generators/DecayerPythia8Param.h" +#include #endif #include "generator_pythia8_longlived.C" @@ -178,12 +179,12 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 if (!pythiaObjectSignal.next()) { continue; } - //Check if triggered condition satisfied + // Check if triggered condition satisfied for (Long_t j = 0; j < pythiaObjectSignal.event.size(); j++) { const int& pypid = pythiaObjectSignal.event[j].id(); const float& pyeta = pythiaObjectSignal.event[j].eta(); const float& pypt = pythiaObjectSignal.event[j].pT(); - if (pypid == cfg.pdg && cfg.etaMin < pyeta && pyeta < cfg.etaMax && pypt > cfg.ptMin && pypt < cfg.ptMax) { + if (pypid == cfg.mPdg && cfg.mEtaMin < pyeta && pyeta < cfg.mEtaMax && pypt > cfg.mPtMin && pypt < cfg.mPtMax) { LOG(info) << "Found particle " << j << " " << pypid << " with eta " << pyeta << " and pT " << pypt << " in event " << mEventCounter << " after " << nTries << " tries"; satisfiesTrigger = true; break; @@ -204,19 +205,19 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 continue; } // Do the injection - for (int i{0}; i < cfg.nInject; ++i) { - const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); - const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); + for (int i{0}; i < cfg.mNInject; ++i) { + const double pt = gRandom->Uniform(cfg.mPtMin, cfg.mPtMax); + const double eta = gRandom->Uniform(cfg.mEtaMin, cfg.mEtaMax); const double phi = gRandom->Uniform(0, TMath::TwoPi()); const double px{pt * std::cos(phi)}; const double py{pt * std::sin(phi)}; const double pz{pt * std::sinh(eta)}; - const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; + const double et{std::hypot(std::hypot(pt, pz), cfg.mMass)}; Particle particle; - particle.id(cfg.pdg); + particle.id(cfg.mPdg); particle.status(11); - particle.m(cfg.mass); + particle.m(cfg.mMass); particle.px(px); particle.py(py); particle.pz(pz); @@ -224,7 +225,7 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 particle.xProd(0.f); particle.yProd(0.f); particle.zProd(0.f); - mPythia.particleData.mayDecay(cfg.pdg, true); // force decay + mPythia.particleData.mayDecay(cfg.mPdg, true); // force decay mPythia.event.append(particle); } injectedForThisEvent = true; @@ -270,16 +271,16 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 if (mConfigToUse >= 0 && (nConfig - 1) != mConfigToUse) { continue; } - LOGF(info, "Injecting %i particles with PDG %i, pT in [%f, %f]", cfg.nInject, cfg.pdg, cfg.ptMin, cfg.ptMax); + LOGF(info, "Injecting %i particles with PDG %i, pT in [%f, %f]", cfg.mNInject, cfg.mPdg, cfg.mPtMin, cfg.mPtMax); - for (int i{0}; i < cfg.nInject; ++i) { - const double pt = gRandom->Uniform(cfg.ptMin, cfg.ptMax); - const double eta = gRandom->Uniform(cfg.etaMin, cfg.etaMax); + for (int i{0}; i < cfg.mNInject; ++i) { + const double pt = gRandom->Uniform(cfg.mPtMin, cfg.mPtMax); + const double eta = gRandom->Uniform(cfg.mEtaMin, cfg.mEtaMax); const double phi = gRandom->Uniform(0, TMath::TwoPi()); const double px{pt * std::cos(phi)}; const double py{pt * std::sin(phi)}; const double pz{pt * std::sinh(eta)}; - const double et{std::hypot(std::hypot(pt, pz), cfg.mass)}; + const double et{std::hypot(std::hypot(pt, pz), cfg.mMass)}; // TParticle::TParticle(Int_t pdg, // Int_t status, @@ -288,7 +289,7 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 // Double_t px, Double_t py, Double_t pz, Double_t etot, // Double_t vx, Double_t vy, Double_t vz, Double_t time) - mParticles.push_back(TParticle(cfg.pdg, + mParticles.push_back(TParticle(cfg.mPdg, MCGenStatusEncoding(1, 1).fullEncoding, -1, -1, -1, -1, @@ -311,65 +312,102 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 } struct ConfigContainer { - ConfigContainer(int input_pdg = 0, int n = 1, float p = 1, float P = 10) : pdg{input_pdg}, - nInject{n}, - ptMin{p}, - ptMax{P} + ConfigContainer(int input_pdg = 0, int n = 1, + float ptMin = 1, float ptMax = 10, + float etaMin = -1, float etaMax = 1) : mPdg{input_pdg}, + mNInject{n}, + mPtMin{ptMin}, + mPtMax{ptMax}, + mEtaMin{etaMin}, + mEtaMax{etaMax} { - mass = GeneratorPythia8LongLivedGun::getMass(pdg); - if (mass <= 0) { - LOG(fatal) << "Could not find mass for pdg " << pdg; + mMass = GeneratorPythia8LongLivedGun::getMass(mPdg); + if (mMass <= 0) { + LOG(fatal) << "Could not find mass for mPdg " << mPdg; } - LOGF(info, "ConfigContainer: pdg = %i, nInject = %i, ptMin = %f, ptMax = %f, mass = %f", pdg, nInject, ptMin, ptMax, mass); + LOGF(info, "ConfigContainer: mPdg = %i, mNInject = %i, mPtMin = %f, mPtMax = %f, mEtaMin = %f, mEtaMax = %f, mMass = %f", + mPdg, mNInject, mPtMin, mPtMax, mEtaMin, mEtaMax, mMass); }; + ConfigContainer(TObjArray* arr) : ConfigContainer(atoi(arr->At(0)->GetName()), atoi(arr->At(1)->GetName()), atof(arr->At(2)->GetName()), - atof(arr->At(3)->GetName())){}; - - int pdg = 0; - int nInject = 1; - float ptMin = 1; - float ptMax = 10; - float etaMin = -1.f; - float etaMax = 1.f; - double mass = 0.f; + atof(arr->At(3)->GetName()), + atof(arr->At(4)->GetName()), + atof(arr->At(5)->GetName())) + { + bool hasGenDecayed = false; + for (int i = 0; i < arr->GetEntries(); i++) { + const TString n = arr->At(i)->GetName(); + std::cout << n << std::endl; + if (n == "genDecayed") { + hasGenDecayed = true; + break; + } + } + if (hasGenDecayed) { + if (arr->GetEntries() != 7) { + LOG(fatal) << "Wrong number of entries in the configuration array, should be 7, is " << arr->GetEntries(); + } + } else { + if (arr->GetEntries() != 6) { + LOG(fatal) << "Wrong number of entries in the configuration array, should be 6, is " << arr->GetEntries(); + } + } + }; + ConfigContainer(TString line) : ConfigContainer(line.Tokenize(" ")){}; + ConfigContainer(const nlohmann::json& jsonParams) : ConfigContainer(jsonParams["pdg"], + jsonParams["n"], + jsonParams["ptMin"], + jsonParams["ptMax"], + jsonParams["etaMin"], + jsonParams["etaMax"]){}; + + // Data Members + const int mPdg = 0; + const int mNInject = 1; + const float mPtMin = 1; + const float mPtMax = 10; + const float mEtaMin = -1.f; + const float mEtaMax = 1.f; + double mMass = 0.f; + void print() const { - LOGF(info, "int pdg = %i", pdg); - LOGF(info, "int nInject = %i", nInject); - LOGF(info, "float ptMin = %f", ptMin); - LOGF(info, "float ptMax = %f", ptMax); - LOGF(info, "float etaMin = %f", etaMin); - LOGF(info, "float etaMax = %f", etaMax); - LOGF(info, "double mass = %f", mass); + LOGF(info, "int mPdg = %i", mPdg); + LOGF(info, "int mNInject = %i", mNInject); + LOGF(info, "float mPtMin = %f", mPtMin); + LOGF(info, "float mPtMax = %f", mPtMax); + LOGF(info, "float mEtaMin = %f", mEtaMin); + LOGF(info, "float mEtaMax = %f", mEtaMax); + LOGF(info, "double mMass = %f", mMass); } }; //__________________________________________________________________ - ConfigContainer addGun(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) + ConfigContainer addGun(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10, float etaMin = 1, float etaMax = 10) { if (mUseTriggering) { // If in trigger mode, every particle needs to be generated from pythia - return addGunGenDecayed(input_pdg, nInject, ptMin, ptMax); + return addGunGenDecayed(input_pdg, nInject, ptMin, ptMax, etaMin, etaMax); } - ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; + ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax, etaMin, etaMax}; mGunConfigs.push_back(cfg); return cfg; } //__________________________________________________________________ - ConfigContainer addGun(ConfigContainer cfg) { return addGun(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } + ConfigContainer addGun(ConfigContainer cfg) { return addGun(cfg.mPdg, cfg.mNInject, cfg.mPtMin, cfg.mPtMax, cfg.mEtaMin, cfg.mEtaMax); } //__________________________________________________________________ - ConfigContainer addGunGenDecayed(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10) + ConfigContainer addGunGenDecayed(int input_pdg, int nInject = 1, float ptMin = 1, float ptMax = 10, float etaMin = 1, float etaMax = 10) { - ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax}; + ConfigContainer cfg{input_pdg, nInject, ptMin, ptMax, etaMin, etaMax}; mGunConfigsGenDecayed.push_back(cfg); return cfg; } //__________________________________________________________________ - ConfigContainer addGunGenDecayed(ConfigContainer cfg) { return addGunGenDecayed(cfg.pdg, cfg.nInject, cfg.ptMin, cfg.ptMax); } + ConfigContainer addGunGenDecayed(ConfigContainer cfg) { return addGunGenDecayed(cfg.mPdg, cfg.mNInject, cfg.mPtMin, cfg.mPtMax, cfg.mEtaMin, cfg.mEtaMax); } //__________________________________________________________________ long int getNGuns() const { return mGunConfigs.size() + mGunConfigsGenDecayed.size(); } @@ -466,7 +504,23 @@ FairGenerator* generateLF(std::string configuration = "${O2DPG_ROOT}/MC/config/P std::ifstream inputFile(configuration.c_str(), ios::in); std::vector cfgVec; std::vector cfgVecGenDecayed; - if (inputFile.is_open()) { + if (!inputFile.is_open()) { + LOGF(fatal, "Can't open '%s' !", configuration.c_str()); + return nullptr; + } + if (TString(configuration.c_str()).EndsWith(".json")) { // read from JSON file + nlohmann::json paramfile = nlohmann::json::parse(inputFile); + std::cout << "paramfile " << paramfile << std::endl; + for (const auto& param : paramfile) { + std::cout << param << std::endl; + // cfgVecGenDecayed.push_back(GeneratorPythia8LF::ConfigContainer{paramfile[n].template get(), param}); + if (param["genDecayed"]) { + cfgVecGenDecayed.push_back(GeneratorPythia8LF::ConfigContainer{param}); + } else { + cfgVec.push_back(GeneratorPythia8LF::ConfigContainer{param}); + } + } + } else { std::string l; int n = 0; while (getline(inputFile, l)) { @@ -476,20 +530,16 @@ FairGenerator* generateLF(std::string configuration = "${O2DPG_ROOT}/MC/config/P if (line.IsNull() || line.IsWhitespace()) { continue; } - if (line.BeginsWith("#")) { std::cout << "Skipping\n"; continue; } if (line.Contains("genDecayed")) { - cfgVecGenDecayed.push_back(GeneratorPythia8LF::ConfigContainer{line.Tokenize(" ")}); + cfgVecGenDecayed.push_back(GeneratorPythia8LF::ConfigContainer{line}); } else { - cfgVec.push_back(GeneratorPythia8LF::ConfigContainer{line.Tokenize(" ")}); + cfgVec.push_back(GeneratorPythia8LF::ConfigContainer{line}); } } - } else { - LOGF(fatal, "Can't open '%s' !", configuration.c_str()); - return nullptr; } return generateLF(cfgVec, cfgVecGenDecayed, injectOnePDGPerEvent, gapBetweenInjection, useTrigger, pythiaCfgMb, pythiaCfgSignal); } @@ -505,7 +555,7 @@ FairGenerator* generateLFTriggered(std::string configuration = "${O2DPG_ROOT}/MC } ///___________________________________________________________ -void generator_pythia8_LF(bool testInj = true, bool testTrg = false) +void generator_pythia8_LF(bool testInj = true, bool testTrg = false, const char* particleListFile = "cfg.json") { LOG(info) << "Compiled correctly!"; if (!testInj && !testTrg) { @@ -514,7 +564,7 @@ void generator_pythia8_LF(bool testInj = true, bool testTrg = false) // Injected mode if (testInj) { LOG(info) << "Testing the injected mode"; - auto* gen = static_cast(generateLF("/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun")); + auto* gen = static_cast(generateLF(particleListFile)); gen->setVerbose(); gen->Print(); gen->print(); @@ -526,10 +576,10 @@ void generator_pythia8_LF(bool testInj = true, bool testTrg = false) // Triggered mode if (testTrg) { LOG(info) << "Testing the triggered mode"; - GeneratorPythia8LF* gen = static_cast(generateLFTriggered("/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", + GeneratorPythia8LF* gen = static_cast(generateLFTriggered(particleListFile, /*gapBetweenInjection=*/0, - /*pythiaCfgMb=*/"/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/inel136tev.cfg", - /*pythiaCfgSignal=*/"/home/njacazio/alice/O2DPG/MC/config/PWGLF/pythia8/generator/inel136tev.cfg")); + /*pythiaCfgMb=*/"inel136tev.cfg", + /*pythiaCfgSignal=*/"inel136tev.cfg")); gen->setVerbose(); gen->Print(); gen->print(); diff --git a/MC/run/PWGLF/run_StrangenessTriggered_pp900gev.sh b/MC/run/PWGLF/run_StrangenessTriggered_pp900gev.sh new file mode 100755 index 000000000..dc0262c0e --- /dev/null +++ b/MC/run/PWGLF/run_StrangenessTriggered_pp900gev.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- CONFIGURE -------------------------- +export IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE=1 +#export ALICEO2_CCDB_LOCALCACHE=.ccdb + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +SIMENGINE=${SIMENGINE:-TGeant4} +NSIGEVENTS=${NSIGEVENTS:-10} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-50000} +SYSTEM=${SYSTEM:-pp} +ENERGY=${ENERGY:-900} +CFGINIFILE=${CFGINIFILE:-"${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_900gev.ini"} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +echo "NWORKERS = $NWORKERS" + +# create workflow +O2_SIM_WORKFLOW=${O2_SIM_WORKFLOW:-"${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py"} +$O2_SIM_WORKFLOW -eCM ${ENERGY} -col ${SYSTEM} -gen external \ + -j ${NWORKERS} \ + -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} \ + -confKey "Diamond.width[2]=6." \ + ${SEED} \ + -e ${SIMENGINE} \ + -ini $CFGINIFILE + +# run workflow +O2_SIM_WORKFLOW_RUNNER=${O2_SIM_WORKFLOW_RUNNER:-"${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py"} +$O2_SIM_WORKFLOW_RUNNER -f workflow.json -tt aod --cpu-limit $NWORKERS From e2eaeb57b5e937fb8abb744b0adb95d54c689b48 Mon Sep 17 00:00:00 2001 From: ffionda <33261977+ffionda@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:51:03 +0200 Subject: [PATCH 0134/1239] update prompt / non-prompt charmonia simulation for introducing trigger gap (#1284) --- .../generator/GeneratorPromptCharmonia.C | 460 ++++++++++++++++++ ...pythia8_NonPromptSignals_gaptriggered_dq.C | 276 +++++++++++ ...ithInjectedPromptSignals_gaptriggered_dq.C | 92 ++++ ...neratorHF_bbbarToBplus_midy_triggerGap.ini | 20 + ...torHF_bbbar_PsiAndJpsi_midy_triggerGap.ini | 20 + ...InjectedPromptCharmoniaMidy_TriggerGap.ini | 7 + ...GeneratorHF_bbbarToBplus_midy_triggerGap.C | 104 ++++ ...ratorHF_bbbar_PsiAndJpsi_midy_triggerGap.C | 92 ++++ ...r_InjectedPromptCharmoniaMidy_TriggerGap.C | 84 ++++ .../generator/pythia8_inel_triggerGap.cfg | 11 + ...unBeautyToPsiAndJpsi_midy_pp_triggerGap.sh | 21 + .../runBplusToJpsi_midy_pp_triggerGap.sh | 21 + .../runPromptCharmonia_midy_pp_triggerGap.sh | 21 + 13 files changed, 1229 insertions(+) create mode 100644 MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C create mode 100644 MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C create mode 100644 MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini create mode 100755 MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToBplus_midy_triggerGap.C create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.C create mode 100644 MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaMidy_TriggerGap.C create mode 100644 MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg create mode 100755 MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh create mode 100755 MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh create mode 100755 MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh diff --git a/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C new file mode 100644 index 000000000..c92f803dd --- /dev/null +++ b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C @@ -0,0 +1,460 @@ +// +// generators for prompt charmonia considering different cases (prompt jpsi, prompt psi2S, prompt jpsi+psi2S) at midrapidity and forward rapidity +// +// usage: +// Jpsi+Psi2S midy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV()" +// Jpsi midy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV()" +// Psi2S midy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorParamPromptPSiToElectronEvtGen_pp13TeV()" +// Jpsi+Psi2S fwdy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV()" +// Jpsi fwdy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV()" +// Psi2S fwdy: o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=$O2DPG_ROOT/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C;GeneratorExternal.funcName=GeneratorParamPromptPSiToMuonEvtGen_pp13TeV()" +// + +R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +#include "GeneratorCocktail.C" +#include "GeneratorEvtGen.C" + +namespace o2 +{ +namespace eventgen +{ + +class O2_GeneratorParamJpsiMidY : public GeneratorTGenerator +{ + + public: + O2_GeneratorParamJpsiMidY() : GeneratorTGenerator("ParamJpsiMidY") + { + paramJpsi = new GeneratorParam(1, -1, PtJPsipp13TeV, YJPsipp13TeV, V2JPsipp13TeV, IpJPsipp13TeV); + paramJpsi->SetMomentumRange(0., 1.e6); + paramJpsi->SetPtRange(0., 1000.); + paramJpsi->SetYRange(-1.0, 1.0); + paramJpsi->SetPhiRange(0., 360.); + paramJpsi->SetDecayer(new TPythia6Decayer()); // Pythia + paramJpsi->SetForceDecay(kNoDecay); // particle left undecayed + setTGenerator(paramJpsi); + }; + + ~O2_GeneratorParamJpsiMidY() + { + delete paramJpsi; + }; + + Bool_t Init() override + { + GeneratorTGenerator::Init(); + paramJpsi->Init(); + return true; + } + + void SetNSignalPerEvent(Int_t nsig) { paramJpsi->SetNumberParticles(nsig); } + + //-------------------------------------------------------------------------// + static Double_t PtJPsipp13TeV(const Double_t* px, const Double_t* /*dummy*/) + { + // prompt J/Psi pT + // pp, 13TeV (tuned on pp 13 TeV, 2016-2018) + // + const Double_t kC = 2.28550e+00; + const Double_t kpt0 = 3.73619e+00; + const Double_t kn = 2.81708e+00; + Double_t pt = px[0]; + + return kC * pt / TMath::Power((1. + (pt / kpt0) * (pt / kpt0)), kn); + } + + //-------------------------------------------------------------------------// + static Double_t YJPsipp13TeV(const Double_t* py, const Double_t* /*dummy*/) + { + // jpsi y in pp at 13 TeV, tuned on data, prompt jpsi ALICE+LHCb, 13 TeV + Double_t y = *py; + Float_t p0, p1, p2; + p0 = 7.79382e+00; + p1 = 2.87827e-06; + p2 = 4.41847e+00; + return p0 * TMath::Exp(-(1. / 2.) * TMath::Power(((y - p1) / p2), 2)); + } + + //-------------------------------------------------------------------------// + static Double_t V2JPsipp13TeV(const Double_t* /*dummy*/, const Double_t* /*dummy*/) + { + // jpsi v2 + return 0.; + } + + //-------------------------------------------------------------------------// + static Int_t IpJPsipp13TeV(TRandom*) + { + return 443; + } + + private: + GeneratorParam* paramJpsi = nullptr; +}; + +class O2_GeneratorParamPsiMidY : public GeneratorTGenerator +{ + + public: + O2_GeneratorParamPsiMidY() : GeneratorTGenerator("ParamPsi") + { + paramPsi = new GeneratorParam(1, -1, PtPsipp13TeV, YPsipp13TeV, V2Psipp13TeV, IpPsipp13TeV); + paramPsi->SetMomentumRange(0., 1.e6); // Momentum range added from me + paramPsi->SetPtRange(0., 1000.); // transverse of momentum range + paramPsi->SetYRange(-1.0, 1.0); // rapidity range + paramPsi->SetPhiRange(0., 360.); // phi range + paramPsi->SetDecayer(new TPythia6Decayer()); // Pythia decayer + paramPsi->SetForceDecay(kNoDecay); // particle left undecayed + setTGenerator(paramPsi); // Setting parameters to ParamPsi for Psi(2S) + }; + + ~O2_GeneratorParamPsiMidY() + { + delete paramPsi; + }; + + Bool_t Init() override + { + GeneratorTGenerator::Init(); + paramPsi->Init(); + return true; + } + void SetNSignalPerEvent(Int_t nsig) { paramPsi->SetNumberParticles(nsig); } + + //-------------------------------------------------------------------------// + static Double_t PtPsipp13TeV(const Double_t* px, const Double_t* /*dummy*/) + { + // prompt J/Psi pT + // pp, 13TeV (tuned on pp 13 TeV, 2016-2018) + // + const Double_t kC = 2.28550e+00; + const Double_t kpt0 = 3.73619e+00; + const Double_t kn = 2.81708e+00; + Double_t pt = px[0]; + + return kC * pt / TMath::Power((1. + (pt / kpt0) * (pt / kpt0)), kn); + } + + //-------------------------------------------------------------------------// + static Double_t YPsipp13TeV(const Double_t* py, const Double_t* /*dummy*/) + { + // jpsi y in pp at 13 TeV, tuned on data, prompt jpsi ALICE+LHCb, 13 TeV + Double_t y = *py; + Float_t p0, p1, p2; + p0 = 7.79382e+00; + p1 = 2.87827e-06; + p2 = 4.41847e+00; + return p0 * TMath::Exp(-(1. / 2.) * TMath::Power(((y - p1) / p2), 2)); + } + + //-------------------------------------------------------------------------// + static Double_t V2Psipp13TeV(const Double_t* /*dummy*/, const Double_t* /*dummy*/) + { + // jpsi v2 + return 0.; + } + + //-------------------------------------------------------------------------// + static Int_t IpPsipp13TeV(TRandom*) + { + return 100443; + } + + private: + GeneratorParam* paramPsi = nullptr; +}; + +class O2_GeneratorParamJpsiFwdY : public GeneratorTGenerator +{ + + public: + O2_GeneratorParamJpsiFwdY() : GeneratorTGenerator("ParamJpsi") + { + paramJpsi = new GeneratorParam(1, -1, PtJPsipp13TeV, YJPsipp13TeV, V2JPsipp13TeV, IpJPsipp13TeV); + paramJpsi->SetMomentumRange(0., 1.e6); + paramJpsi->SetPtRange(0, 999.); + paramJpsi->SetYRange(-4.2, -2.3); + paramJpsi->SetPhiRange(0., 360.); + paramJpsi->SetDecayer(new TPythia6Decayer()); + paramJpsi->SetForceDecay(kNoDecay); // particle left undecayed + // - - paramJpsi->SetTrackingFlag(1); // (from AliGenParam) -> check this + setTGenerator(paramJpsi); + }; + + ~O2_GeneratorParamJpsiFwdY() + { + delete paramJpsi; + }; + + Bool_t Init() override + { + GeneratorTGenerator::Init(); + paramJpsi->Init(); + return true; + } + + void SetNSignalPerEvent(Int_t nsig) { paramJpsi->SetNumberParticles(nsig); } + + //-------------------------------------------------------------------------// + static Double_t PtJPsipp13TeV(const Double_t* px, const Double_t* /*dummy*/) + { + // jpsi pT in pp at 13 TeV, tuned on data (2015) + Double_t x = *px; + Float_t p0, p1, p2, p3; + p0 = 1; + p1 = 4.75208; + p2 = 1.69247; + p3 = 4.49224; + return p0 * x / TMath::Power(1. + TMath::Power(x / p1, p2), p3); + } + + //-------------------------------------------------------------------------// + static Double_t YJPsipp13TeV(const Double_t* py, const Double_t* /*dummy*/) + { + // jpsi y in pp at 13 TeV, tuned on data (2015) + Double_t y = *py; + Float_t p0, p1, p2; + p0 = 1; + p1 = 0; + p2 = 2.98887; + return p0 * TMath::Exp(-(1. / 2.) * TMath::Power(((y - p1) / p2), 2)); + } + + //-------------------------------------------------------------------------// + static Double_t V2JPsipp13TeV(const Double_t* /*dummy*/, const Double_t* /*dummy*/) + { + // jpsi v2 + return 0.; + } + + //-------------------------------------------------------------------------// + static Int_t IpJPsipp13TeV(TRandom*) + { + return 443; + } + + private: + GeneratorParam* paramJpsi = nullptr; +}; + +class O2_GeneratorParamPsiFwdY : public GeneratorTGenerator +{ + + public: + O2_GeneratorParamPsiFwdY() : GeneratorTGenerator("ParamPsi") + { + paramPsi = new GeneratorParam(1, -1, PtPsipp13TeV, YPsipp13TeV, V2Psipp13TeV, IpPsipp13TeV); + paramPsi->SetMomentumRange(0., 1.e6); + paramPsi->SetPtRange(0, 999.); + paramPsi->SetYRange(-4.2, -2.3); + paramPsi->SetPhiRange(0., 360.); + paramPsi->SetDecayer(new TPythia6Decayer()); + paramPsi->SetForceDecay(kNoDecay); // particle left undecayed + // - - paramJpsi->SetTrackingFlag(1); // check this + setTGenerator(paramPsi); + }; + + ~O2_GeneratorParamPsiFwdY() + { + delete paramPsi; + }; + + Bool_t Init() override + { + GeneratorTGenerator::Init(); + paramPsi->Init(); + return true; + } + + void SetNSignalPerEvent(Int_t nsig) { paramPsi->SetNumberParticles(nsig); } + + //-------------------------------------------------------------------------// + static Double_t PtPsipp13TeV(const Double_t* px, const Double_t* /*dummy*/) + { + // jpsi pT in pp at 13 TeV, tuned on data (2015) + Double_t x = *px; + Float_t p0, p1, p2, p3; + p0 = 1; + p1 = 4.75208; + p2 = 1.69247; + p3 = 4.49224; + return p0 * x / TMath::Power(1. + TMath::Power(x / p1, p2), p3); + } + + //-------------------------------------------------------------------------// + static Double_t YPsipp13TeV(const Double_t* py, const Double_t* /*dummy*/) + { + // jpsi y in pp at 13 TeV, tuned on data (2015) + Double_t y = *py; + Float_t p0, p1, p2; + p0 = 1; + p1 = 0; + p2 = 2.98887; + return p0 * TMath::Exp(-(1. / 2.) * TMath::Power(((y - p1) / p2), 2)); + } + + //-------------------------------------------------------------------------// + static Double_t V2Psipp13TeV(const Double_t* /*dummy*/, const Double_t* /*dummy*/) + { + // jpsi v2 + return 0.; + } + + //-------------------------------------------------------------------------// + static Int_t IpPsipp13TeV(TRandom*) + { + return 100443; + } + + private: + GeneratorParam* paramPsi = nullptr; +}; + + +} // namespace eventgen +} // namespace o2 + +FairGenerator* + GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV() +{ + auto genCocktailEvtGen = new o2::eventgen::GeneratorEvtGen(); + + auto genJpsi = new o2::eventgen::O2_GeneratorParamJpsiMidY; + genJpsi->SetNSignalPerEvent(1); // signal per event for J/Psi + auto genPsi = new o2::eventgen::O2_GeneratorParamPsiMidY; + genPsi->SetNSignalPerEvent(1); // signal per event for Psi(2s) + genCocktailEvtGen->AddGenerator(genJpsi, 1); // add cocktail --> J/Psi + genCocktailEvtGen->AddGenerator(genPsi, 1); // add cocktail --> Psi(2s) + + TString pdgs = "443;100443"; + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + genCocktailEvtGen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + genCocktailEvtGen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + genCocktailEvtGen->SetForceDecay(kEvtDiElectron); + + // print debug + genCocktailEvtGen->PrintDebug(); + + return genCocktailEvtGen; +} + +FairGenerator* + GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV(TString pdgs = "443") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->SetNSignalPerEvent(1); // number of jpsis per event + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtDiElectron); + + // print debug + gen->PrintDebug(); + + return gen; +} + +FairGenerator* + GeneratorParamPromptPsiToElectronEvtGen_pp13TeV(TString pdgs = "100443") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->SetNSignalPerEvent(1); // number of jpsis per event + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtDiElectron); + + // print debug + gen->PrintDebug(); + + return gen; +} + + +FairGenerator* GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV() +{ + + auto genCocktailEvtGen = new o2::eventgen::GeneratorEvtGen(); + + auto genJpsi = new o2::eventgen::O2_GeneratorParamJpsiFwdY; + genJpsi->SetNSignalPerEvent(4); // 4 J/psi generated per event by GeneratorParam + auto genPsi = new o2::eventgen::O2_GeneratorParamPsiFwdY; + genPsi->SetNSignalPerEvent(2); // 2 Psi(2S) generated per event by GeneratorParam + genCocktailEvtGen->AddGenerator(genJpsi, 1); // 2/3 J/psi + genCocktailEvtGen->AddGenerator(genPsi, 1); // 1/3 Psi(2S) + + TString pdgs = "443;100443"; + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + genCocktailEvtGen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + genCocktailEvtGen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + genCocktailEvtGen->SetForceDecay(kEvtDiMuon); + + return genCocktailEvtGen; +} + +FairGenerator* + GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV(TString pdgs = "443") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->SetNSignalPerEvent(1); // number of jpsis per event + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtDiMuon); + + // print debug + gen->PrintDebug(); + + return gen; +} + +FairGenerator* + GeneratorParamPromptPsiToMuonEvtGen_pp13TeV(TString pdgs = "100443") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->SetNSignalPerEvent(1); // number of jpsis per event + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtDiMuon); + + // print debug + gen->PrintDebug(); + + return gen; +} + diff --git a/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C b/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C new file mode 100644 index 000000000..8c5859997 --- /dev/null +++ b/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C @@ -0,0 +1,276 @@ +#include "FairGenerator.h" +#include "Generators/GeneratorPythia8.h" +#include "Pythia8/Pythia.h" +#include "TRandom.h" + +R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +#include "GeneratorEvtGen.C" + +#include + +using namespace o2::eventgen; +using namespace Pythia8; + +namespace o2 +{ +namespace eventgen +{ + +class GeneratorPythia8NonPromptInjectedGapTriggeredDQ : public o2::eventgen::GeneratorPythia8 { +public: + + /// constructor + GeneratorPythia8NonPromptInjectedGapTriggeredDQ(int inputTriggerRatio = 5) { + + mGeneratedEvents = 0; + mInverseTriggerRatio = inputTriggerRatio; + // define minimum bias event generator + auto seed = (gRandom->TRandom::GetSeed() % 900000000); + TString pathconfigMB = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg"); + pythiaMBgen.readFile(pathconfigMB.Data()); + pythiaMBgen.readString("Random:setSeed on"); + pythiaMBgen.readString("Random:seed " + std::to_string(seed)); + mConfigMBdecays = ""; + mPDG = 5; + mRapidityMin = -1.; + mRapidityMax = 1.; + mHadronMultiplicity = -1; + mHadronRapidityMin = -1.; + mHadronRapidityMax = 1.; + mVerbose = false; + } + + /// Destructor + ~GeneratorPythia8NonPromptInjectedGapTriggeredDQ() = default; + + void setPDG(int val) { mPDG = val; }; + void addHadronPDGs(int pdg) { mHadronsPDGs.push_back(pdg); }; + void setHadronMultiplicity(int val) { mHadronMultiplicity = val; }; + void setRapidity(double valMin, double valMax) + { + mRapidityMin = valMin; + mRapidityMax = valMax; + }; + + void setRapidityHadron(double valMin, double valMax) + { + mHadronRapidityMin = valMin; + mHadronRapidityMax = valMax; + }; + + void setConfigMBdecays(TString val){mConfigMBdecays = val;} + + void setVerbose(bool val) { mVerbose = val; }; + +protected: + +Bool_t generateEvent() override + { + // reset event + bool genOk = false; + if (mGeneratedEvents % mInverseTriggerRatio == 0){ + bool ancestor = false; + while (! (genOk && ancestor) ){ + /// reset event + mPythia.event.reset(); + genOk = GeneratorPythia8::generateEvent(); + // find the q-qbar ancestor + ancestor = findHeavyQuarkPair(mPythia.event); + } + } else { + /// reset event + pythiaMBgen.event.reset(); + while (!genOk) { + genOk = pythiaMBgen.next(); + } + mPythia.event = pythiaMBgen.event; + } + mGeneratedEvents++; + if (mVerbose) mOutputEvent.list(); + return true; + } + +Bool_t Init() override + { + if(mConfigMBdecays.Contains("cfg")) pythiaMBgen.readFile(mConfigMBdecays.Data()); + GeneratorPythia8::Init(); + pythiaMBgen.init(); + return true; + } + + // search for q-qbar mother with at least one q in a selected rapidity window + bool findHeavyQuarkPair(Pythia8::Event& event) + { + int countH[mHadronsPDGs.size()]; for(int ipdg=0; ipdg < mHadronsPDGs.size(); ipdg++) countH[ipdg]=0; + bool hasq = false, hasqbar = false, atSelectedY = false, isOkAtPartonicLevel = false; + for (int ipa = 0; ipa < event.size(); ++ipa) { + + if(!isOkAtPartonicLevel){ + auto daughterList = event[ipa].daughterList(); + hasq = false; hasqbar = false; atSelectedY = false; + for (auto ida : daughterList) { + if (event[ida].id() == mPDG) + hasq = true; + if (event[ida].id() == -mPDG) + hasqbar = true; + if ((event[ida].y() > mRapidityMin) && (event[ida].y() < mRapidityMax)) + atSelectedY = true; + } + if (hasq && hasqbar && atSelectedY) isOkAtPartonicLevel = true; + } + + if( (mHadronMultiplicity <= 0) && isOkAtPartonicLevel) return true; // no selection at hadron level + + /// check at hadron level if needed + int ipdg=0; + for (auto& pdgVal : mHadronsPDGs){ + if ( (TMath::Abs(event[ipa].id()) == pdgVal) && (event[ipa].y() > mHadronRapidityMin) && (event[ipa].y() < mHadronRapidityMax) ) { countH[ipdg]++; printf("ndaughters %d \n",event[ipa].daughterList().size()); } + if(isOkAtPartonicLevel && countH[ipdg] >= mHadronMultiplicity) return true; + ipdg++; + } + } + return false; + }; + + +private: +// Interface to override import particles +Pythia8::Event mOutputEvent; + + // Control gap-triggering + unsigned long long mGeneratedEvents; + int mInverseTriggerRatio; + Pythia pythiaMBgen; // minimum bias event + TString mConfigMBdecays; + int mPDG; + std::vector mHadronsPDGs; + int mHadronMultiplicity; + double mRapidityMin; + double mRapidityMax; + double mHadronRapidityMin; + double mHadronRapidityMax; + bool mVerbose; + }; + +} + +} + +// Predefined generators: +FairGenerator* + GeneratorBeautyToJpsi_EvtGenMidY(double rapidityMin = -1.5, double rapidityMax = 1.5, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + gen->setRapidityHadron(-1.5,1.5); + gen->setHadronMultiplicity(1); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + gen->setConfigMBdecays(pathO2table); + gen->setVerbose(verbose); + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + gen->addHadronPDGs(std::stoi(spdg)); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBJpsiDiElectron); + + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + // print debug + // gen->PrintDebug(); + + return gen; +} + +FairGenerator* + GeneratorBeautyToPsiAndJpsi_EvtGenMidY(double rapidityMin = -1.5, double rapidityMax = 1.5, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + gen->setRapidityHadron(rapidityMin,rapidityMax); + gen->setHadronMultiplicity(1); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + gen->setConfigMBdecays(pathO2table); + gen->setVerbose(verbose); + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + gen->addHadronPDGs(std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBPsiAndJpsiDiElectron); + + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + // print debug + // gen->PrintDebug(); + + return gen; +} + +FairGenerator* + GeneratorBplusToJpsiKaon_EvtGen(double rapidityMin = -1.5, double rapidityMax = 1.5, bool verbose = false, TString pdgs = "521") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + gen->addHadronPDGs(521); + gen->setRapidityHadron(rapidityMin,rapidityMax); + gen->setHadronMultiplicity(2); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBplus.cfg"); + gen->readFile(pathO2table.Data()); + gen->setConfigMBdecays(pathO2table); + gen->setVerbose(verbose); + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + gen->addHadronPDGs(std::stoi(spdg)); + printf("PDG %d \n", std::stoi(spdg)); + } + + TString pathO2 = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen"); + gen->SetDecayTable(Form("%s/BPLUSTOKAONJPSITOELE.DEC", pathO2.Data())); + // print debug + // gen->PrintDebug(); + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + return gen; +} + + + diff --git a/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C b/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C new file mode 100644 index 000000000..5901729f4 --- /dev/null +++ b/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C @@ -0,0 +1,92 @@ +#include "FairGenerator.h" +#include "Generators/GeneratorPythia8.h" +#include "Pythia8/Pythia.h" +#include "TRandom.h" +#include "GeneratorPromptCharmonia.C" + +#include + +using namespace o2::eventgen; +using namespace Pythia8; + +class GeneratorPythia8PromptInjectedGapTriggeredDQ : public o2::eventgen::GeneratorPythia8 { +public: + + /// default constructor + GeneratorPythia8PromptInjectedGapTriggeredDQ() = default; + + /// constructor + GeneratorPythia8PromptInjectedGapTriggeredDQ(int inputTriggerRatio = 5, int gentype = 0) { + + mGeneratedEvents = 0; + mGeneratorParam = 0x0; + mInverseTriggerRatio = inputTriggerRatio; + switch (gentype) { + case 0: // generate prompt jpsi + psi2s cocktail at midrapidity + mGeneratorParam = (Generator*)GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV(); + break; + case 1: // generate prompt jpsi at midrapidity + mGeneratorParam = (Generator*)GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV("443"); + break; + case 2: // generate prompt psi2S at midrapidity + mGeneratorParam = (Generator*)GeneratorParamPromptPsiToElectronEvtGen_pp13TeV("100443"); + break; + case 3: // generate prompt jpsi + psi2s cocktail at forward rapidity + mGeneratorParam = (Generator*)GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV(); + break; + case 4: // generate prompt jpsi at forward rapidity + mGeneratorParam = (Generator*)GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV("443"); + break; + case 5: // generate prompt psi2S at forward rapidity + mGeneratorParam = (Generator*)GeneratorParamPromptPsiToMuonEvtGen_pp13TeV("100443"); + break; + } + mGeneratorParam->Init(); + } + + /// Destructor + ~GeneratorPythia8PromptInjectedGapTriggeredDQ() = default; + +protected: + +Bool_t importParticles() override + { + GeneratorPythia8::importParticles(); + bool genOk = false; + if (mGeneratedEvents % mInverseTriggerRatio == 0){ // add injected prompt signals to the stack + bool genOk = false; + while (!genOk){ + genOk = (mGeneratorParam->generateEvent() && mGeneratorParam->importParticles()) ? true : false ; + } + int originalSize = mParticles.size(); + for(int ipart=0; ipart < mGeneratorParam->getParticles().size(); ipart++){ + TParticle part = TParticle(mGeneratorParam->getParticles().at(ipart)); + if(part.GetFirstMother() >= 0) part.SetFirstMother(part.GetFirstMother() + originalSize); + if(part.GetFirstDaughter() >= 0) part.SetFirstDaughter(part.GetFirstDaughter() + originalSize); + if(part.GetLastDaughter() >= 0) part.SetLastDaughter(part.GetLastDaughter() + originalSize); + mParticles.push_back(part); + // encodeParticleStatusAndTracking method already called in GeneratorEvtGen.C + } + mGeneratorParam->clearParticles(); + } + + mGeneratedEvents++; + return true; + } + + +private: + Generator* mGeneratorParam; + // Control gap-triggering + unsigned long long mGeneratedEvents; + int mInverseTriggerRatio; +}; + +// Predefined generators: +FairGenerator *GeneratorPythia8InjectedPromptCharmoniaGapTriggered(int inputTriggerRatio, int gentype) { + auto myGen = new GeneratorPythia8PromptInjectedGapTriggeredDQ(inputTriggerRatio,gentype); + auto seed = (gRandom->TRandom::GetSeed() % 900000000); + myGen->readString("Random:setSeed on"); + myGen->readString("Random:seed " + std::to_string(seed)); + return myGen; +} diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini new file mode 100644 index 000000000..69f0d4a2b --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini @@ -0,0 +1,20 @@ +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C +funcName = GeneratorBplusToJpsiKaon_EvtGen() + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-1.5,1.5) + +### The setup inhibits transport of primary particles which are produce at forward rapidity. +### The settings below only transports particles in the barrel, which is currently defined as |eta| < 2 + +[Stack] +transportPrimary = barrel diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini new file mode 100644 index 000000000..f014af9c8 --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini @@ -0,0 +1,20 @@ +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C +funcName = GeneratorBeautyToPsiAndJpsi_EvtGenMidY() + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-1.5,1.5) + +### The setup inhibits transport of primary particles which are produce at forward rapidity. +### The settings below only transports particles in the barrel, which is currently defined as |eta| < 2 + +[Stack] +transportPrimary = barrel diff --git a/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini b/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini new file mode 100755 index 000000000..f2ed68c40 --- /dev/null +++ b/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C +funcName=GeneratorPythia8InjectedPromptCharmoniaGapTriggered(5,0) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToBplus_midy_triggerGap.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToBplus_midy_triggerGap.C new file mode 100644 index 000000000..fe282e809 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToBplus_midy_triggerGap.C @@ -0,0 +1,104 @@ +int External() +{ + int checkPdgSignal[] = {443}; + int checkPdgDecay = 11; + int kaonPdg = 321; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal[0] << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalKaons{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalKaonsWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Int_t bpdgs[] = {521}; + Int_t sizePdg = sizeof(bpdgs)/sizeof(Int_t); + Bool_t hasBeautyMoth = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0]) { + // check if mothers are beauty hadrons + hasBeautyMoth = kFALSE; + if(idMoth){ // check beauty mother + auto tdM = mcreader.getTrack(i, idMoth); + for(int i=0; iGetPdgCode()) == bpdgs[i] ) hasBeautyMoth = kTRUE; } + // check that it has 2 daughters, Jpsi + K + auto child0b = o2::mcutils::MCTrackNavigator::getDaughter0(*tdM, *tracks); + auto child1b = o2::mcutils::MCTrackNavigator::getDaughter1(*tdM, *tracks); + if (child0b != nullptr && child1b != nullptr) { + auto pdg0b = child0b->GetPdgCode(); + auto pdg1b = child1b->GetPdgCode(); + std::cout << "First and last children of parent B+ " << tdM->GetPdgCode() << " are PDG0: " << pdg0b << " PDG1: " << pdg1b << "\n"; + if(TMath::Abs(pdg0b) == kaonPdg ) { nSignalKaons++; if( std::abs(track.GetRapidity()) < 1.5) nSignalKaonsWithinAcc++; } + if(TMath::Abs(pdg1b) == kaonPdg ) { nSignalKaons++; if( std::abs(track.GetRapidity()) < 1.5) nSignalKaonsWithinAcc++; } + } + } + if(hasBeautyMoth){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if( (std::abs(rapidity) < 1.5) && pdg == checkPdgSignal[0] ) nSignalJpsiWithinAcc++; + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal[0] << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (jpsi <- B+): " << nSignalJpsi << "; within acceptance (|y| < 1.5): " << nSignalJpsiWithinAcc << "\n" + << "#signal (K+ <- B+): " << nSignalKaons << "; within acceptance (|y| < 1.5): " << nSignalKaonsWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.C new file mode 100644 index 000000000..8a0964910 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.C @@ -0,0 +1,92 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 11; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Int_t bpdgs[] = {511, 521, 531, 5112, 5122, 5232, 5132}; + Int_t sizePdg = sizeof(bpdgs)/sizeof(Int_t); + Bool_t hasBeautyMoth = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + // check if mothers are beauty hadrons + hasBeautyMoth = kFALSE; + if(idMoth){ // check beauty mother + auto tdM = mcreader.getTrack(i, idMoth); + for(int i=0; iGetPdgCode()) == bpdgs[i] ) hasBeautyMoth = kTRUE; } + } + if(hasBeautyMoth){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(std::abs(rapidity) < 1.0) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (jpsi <- b): " << nSignalJpsi << "; within acceptance (|y| < 1): " << nSignalJpsiWithinAcc << "\n" + << "#signal (psi2S <- b): " << nSignalPsi2S << "; within acceptance (|y| < 1): " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaMidy_TriggerGap.C b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaMidy_TriggerGap.C new file mode 100644 index 000000000..b155ec9ac --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaMidy_TriggerGap.C @@ -0,0 +1,84 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 11; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Bool_t isInjected = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + if(idMoth < 0){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(std::abs(rapidity) < 1.0) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (prompt Jpsi): " << nSignalJpsi << "; within acceptance (|y| < 1): " << nSignalJpsiWithinAcc << "\n" + << "#signal (prompt Psi(2S)): " << nSignalPsi2S << "; within acceptance (|y| < 1): " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg b/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg new file mode 100644 index 000000000..a4c535826 --- /dev/null +++ b/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg @@ -0,0 +1,11 @@ +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +SoftQCD:inelastic on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. diff --git a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh new file mode 100755 index 000000000..145c29d6f --- /dev/null +++ b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini --mft-reco-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh new file mode 100755 index 000000000..0e74920d8 --- /dev/null +++ b/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini --mft-reco-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh new file mode 100755 index 000000000..0636e7eb6 --- /dev/null +++ b/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini --mft-reco-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json From 4c16774ed6d584c27baa263ed7a50f25b5b38777 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 19 Oct 2023 23:50:29 +0200 Subject: [PATCH 0135/1239] Add TRD krypton run workflow (#1286) --- DATA/testing/detectors/TRD/trd-krypton.sh | 51 +++++++++++++++++++++++ DATA/testing/detectors/TRD/workflows.desc | 1 + 2 files changed, 52 insertions(+) create mode 100644 DATA/testing/detectors/TRD/trd-krypton.sh diff --git a/DATA/testing/detectors/TRD/trd-krypton.sh b/DATA/testing/detectors/TRD/trd-krypton.sh new file mode 100644 index 000000000..fd786f116 --- /dev/null +++ b/DATA/testing/detectors/TRD/trd-krypton.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Make common arguments and helper functions such as add_W available +source common/setenv.sh +source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh + + +# Define input data required by DPL (in this case all RAWDATA from TRD) +PROXY_INSPEC="A:TRD/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" + +# Allow for setting external options + +: ${TRD_N_READERS:=16} +: ${TRD_N_ENCODER:=4} + +: ${EPN2EOS_METAFILES_DIR:="/data/epn2eos_tool/epn2eos"} +: ${CTF_DIR:="/data/tf/compressed"} +: ${CALIB_DIR:="/data/calibration"} + +: ${RANS_OPT:="--ans-version 1.0 --ctf-dict none"} + +: ${CTF_MINSIZE:="10000000000"} # accumulate CTFs until file size reached +: ${CTF_MAX_PER_FILE:="40000"} # but no more than given number of CTFs per file +: ${CTF_FREE_DISK_WAIT:="10"} # if disk on EPNs is close to full, wait X seconds before retrying to write +: ${CTF_MAX_FREE_DISK_WAIT:="600"} # if not enough disk space after this time throw error +: ${CTF_OUTPUT_TYPE:="ctf"} + +CTF_CONFIG="--report-data-size-interval 1000" +CTF_CONFIG+=" --output-dir $CTF_DIR --output-type $CTF_OUTPUT_TYPE --min-file-size $CTF_MINSIZE --max-ctf-per-file ${CTF_MAX_PER_FILE} --onlyDet TRD --meta-output-dir $EPN2EOS_METAFILES_DIR" +CTF_CONFIG+=" --require-free-disk 53687091200 --wait-for-free-disk $CTF_FREE_DISK_WAIT --max-wait-for-free-disk $CTF_MAX_FREE_DISK_WAIT" + + +# Start with an empty workflow +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 +add_W o2-trd-datareader "--disable-root-output --pipeline trd-datareader:$TRD_N_READERS" +add_W o2-trd-kr-clusterer "--meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --autosave-interval 105000 --pipeline trd-kr-clusterer:8" +if workflow_has_parameter QC && has_detector_qc TRD; then + add_QC_from_consul "/o2/components/qc/ANY/any/trd-full-qcmn-test" "--local --host epn -b" +fi +if workflow_has_parameter CTF; then + add_W o2-trd-entropy-encoder-workflow "$RANS_OPT --mem-factor ${TRD_ENC_MEMFACT:-1.5} --pipeline trd-entropy-encoder:$TRD_N_ENCODER" + add_W o2-ctf-writer-workflow "$CONFIG_CTF" +fi + +# Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) +WORKFLOW+="o2-dpl-run $GLOBALDPLOPT $ARGS_ALL" +[[ $WORKFLOWMODE != "print" ]] && WORKFLOW+=" --${WORKFLOWMODE} ${WORKFLOWMODE_FILE:-}" +[[ $WORKFLOWMODE == "print" || "${PRINT_WORKFLOW:-}" == "1" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2') +if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi diff --git a/DATA/testing/detectors/TRD/workflows.desc b/DATA/testing/detectors/TRD/workflows.desc index 859fad1e2..b0624af62 100644 --- a/DATA/testing/detectors/TRD/workflows.desc +++ b/DATA/testing/detectors/TRD/workflows.desc @@ -1 +1,2 @@ TRD-noise: "O2PDPSuite" reco,1,1,"SHMSIZE=$((128 << 30)) testing/detectors/TRD/trd-noise.sh" +TRD-krypton: "O2PDPSuite" reco,40,35,"SHMSIZE=$((128 << 30)) testing/detectors/TRD/trd-kryptona.sh" From 9172589dc48e6fde0ed6e00e8e51679872707b6b Mon Sep 17 00:00:00 2001 From: emeninno <41640762+emeninno@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:18:05 +0200 Subject: [PATCH 0136/1239] few changes made in the PlotOverlays.C, in order to make the legend in the 2D overlay plots more readable (#1293) --- RelVal/PlotOverlays.C | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/RelVal/PlotOverlays.C b/RelVal/PlotOverlays.C index 6ac74d090..f73557544 100644 --- a/RelVal/PlotOverlays.C +++ b/RelVal/PlotOverlays.C @@ -240,7 +240,15 @@ void overlay2D(std::vector hVec1, std::vector labelVec, TLege if (legend){ c.cd(3); - legend->Draw(); + legend->SetTextSize(0.03); + legend->SetTextFont(62); + legend->Draw("same"); + gPad->Update(); + legend->SetX1NDC(0.4); + legend->SetY1NDC(0.7); + legend->SetX2NDC(0.89); + legend->SetY2NDC(0.89); + gPad->Modified(); } auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; From d830cdfe1bf745db4997a9613b49dc7767bf1f05 Mon Sep 17 00:00:00 2001 From: ffionda Date: Mon, 23 Oct 2023 16:31:10 +0200 Subject: [PATCH 0137/1239] add configuration files for fwd-y productions + add other fixes --- .../BPLUSTOKAONJPSITOELEALLMODES.DEC | 49 ++++++++++ .../generator/GeneratorPromptCharmonia.C | 8 +- ...pythia8_NonPromptSignals_gaptriggered_dq.C | 81 +++++++++++++++- ...torHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini | 15 +++ ...InjectedPromptCharmoniaFwdy_TriggerGap.ini | 7 ++ ...ator_InjectedPromptJpsiFwdy_TriggerGap.ini | 7 ++ ...tor_InjectedPromptPsi2SFwdy_TriggerGap.ini | 7 ++ ...ratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.C | 93 +++++++++++++++++++ ...r_InjectedPromptCharmoniaFwdy_TriggerGap.C | 85 +++++++++++++++++ ...erator_InjectedPromptJpsiFwdy_TriggerGap.C | 82 ++++++++++++++++ ...rator_InjectedPromptPsi2SFwdy_TriggerGap.C | 82 ++++++++++++++++ ...unBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh | 21 +++++ ...unBeautyToPsiAndJpsi_midy_pp_triggerGap.sh | 2 +- ...Charmonia_fwdy_pp_assessment_triggerGap.sh | 21 +++++ ...romptJpsi_fwdy_pp_assessment_triggerGap.sh | 21 +++++ ...omptPsi2S_fwdy_pp_assessment_triggerGap.sh | 21 +++++ 16 files changed, 595 insertions(+), 7 deletions(-) create mode 100644 MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BPLUSTOKAONJPSITOELEALLMODES.DEC create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini create mode 100755 MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini create mode 100755 MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini create mode 100755 MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.C create mode 100644 MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.C create mode 100644 MC/config/PWGDQ/ini/tests/Generator_InjectedPromptJpsiFwdy_TriggerGap.C create mode 100644 MC/config/PWGDQ/ini/tests/Generator_InjectedPromptPsi2SFwdy_TriggerGap.C create mode 100755 MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh create mode 100755 MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh create mode 100755 MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh create mode 100755 MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh diff --git a/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BPLUSTOKAONJPSITOELEALLMODES.DEC b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BPLUSTOKAONJPSITOELEALLMODES.DEC new file mode 100644 index 000000000..87da11521 --- /dev/null +++ b/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen/BPLUSTOKAONJPSITOELEALLMODES.DEC @@ -0,0 +1,49 @@ +################## +## Resonant decays not included +Decay B- +# +# B -> cc= s sum = 1.92% +# +#MODIFIED in order to have 50% of B+ decaying in two bodies +0.001914000 J/psi K- SVS; #[Reconstructed PDG2011] +#0.001430000 J/psi K*- SVV_HELAMP PKHminus PKphHminus PKHzero PKphHzero PKHplus PKphHplus; #[Reconstructed PDG2011] +0.0001 J/psi K- pi0 PHSP; +0.000052000 J/psi phi K- PHSP; #[Reconstructed PDG2011] +# +0.001070000 J/psi K- pi- pi+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K- PHSP; #[New mode added] #[Reconstructed PDG2011] +# +#0.0001 J/psi K'_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +#0.0005 J/psi K_2*- PHSP; +#0.001800000 J/psi K_1- SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] + +Enddecay + +Decay B+ +# +# B -> cc= s sum = 1.92% +# +#MODIFIED in order to have 50% of B+ decaying in two bodies +0.001914000 J/psi K+ SVS; #[Reconstructed PDG2011] +#0.001430000 J/psi K*+ SVV_HELAMP PKHplus PKphHplus PKHzero PKphHzero PKHminus PKphHminus; #[Reconstructed PDG2011] +0.0001 J/psi K+ pi0 PHSP; +0.000052000 J/psi phi K+ PHSP; #[Reconstructed PDG2011] +# +0.001070000 J/psi K+ pi+ pi- PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000108000 J/psi eta K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +0.000350000 J/psi omega K+ PHSP; #[New mode added] #[Reconstructed PDG2011] +# +#0.0001 J/psi K'_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; +#0.0005 J/psi K_2*+ PHSP; +#0.001800000 J/psi K_1+ SVV_HELAMP 0.5 0.0 1.0 0.0 0.5 0.0; #[Reconstructed PDG2011] + +Enddecay + +Decay J/psi +1.000 e+ e- PHOTOS VLL; +Enddecay + + +End + diff --git a/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C index c92f803dd..17f3e75af 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C +++ b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C @@ -394,11 +394,11 @@ FairGenerator* GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV() auto genCocktailEvtGen = new o2::eventgen::GeneratorEvtGen(); auto genJpsi = new o2::eventgen::O2_GeneratorParamJpsiFwdY; - genJpsi->SetNSignalPerEvent(4); // 4 J/psi generated per event by GeneratorParam + genJpsi->SetNSignalPerEvent(1); // 1 J/psi generated per event by GeneratorParam auto genPsi = new o2::eventgen::O2_GeneratorParamPsiFwdY; - genPsi->SetNSignalPerEvent(2); // 2 Psi(2S) generated per event by GeneratorParam - genCocktailEvtGen->AddGenerator(genJpsi, 1); // 2/3 J/psi - genCocktailEvtGen->AddGenerator(genPsi, 1); // 1/3 Psi(2S) + genPsi->SetNSignalPerEvent(1); // 1 Psi(2S) generated per event by GeneratorParam + genCocktailEvtGen->AddGenerator(genJpsi, 1); // add J/psi generator + genCocktailEvtGen->AddGenerator(genPsi, 1); // add Psi(2S) generator TString pdgs = "443;100443"; std::string spdg; diff --git a/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C b/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C index 8c5859997..eac21670c 100644 --- a/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C +++ b/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C @@ -124,7 +124,7 @@ Bool_t Init() override /// check at hadron level if needed int ipdg=0; for (auto& pdgVal : mHadronsPDGs){ - if ( (TMath::Abs(event[ipa].id()) == pdgVal) && (event[ipa].y() > mHadronRapidityMin) && (event[ipa].y() < mHadronRapidityMax) ) { countH[ipdg]++; printf("ndaughters %d \n",event[ipa].daughterList().size()); } + if ( (TMath::Abs(event[ipa].id()) == pdgVal) && (event[ipa].y() > mHadronRapidityMin) && (event[ipa].y() < mHadronRapidityMax) ) countH[ipdg]++; if(isOkAtPartonicLevel && countH[ipdg] >= mHadronMultiplicity) return true; ipdg++; } @@ -258,7 +258,8 @@ FairGenerator* } TString pathO2 = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/EvtGen/DecayTablesEvtgen"); - gen->SetDecayTable(Form("%s/BPLUSTOKAONJPSITOELE.DEC", pathO2.Data())); + //gen->SetDecayTable(Form("%s/BPLUSTOKAONJPSITOELE.DEC", pathO2.Data())); + gen->SetDecayTable(Form("%s/BPLUSTOKAONJPSITOELEALLMODES.DEC", pathO2.Data())); // decay table including decay modes for correlated background // print debug // gen->PrintDebug(); // set random seed @@ -272,5 +273,81 @@ FairGenerator* return gen; } +// Predefined generators: +FairGenerator* + GeneratorBeautyToJpsi_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = -2.3, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + gen->setRapidityHadron(rapidityMin,rapidityMax); + gen->setHadronMultiplicity(1); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + gen->setConfigMBdecays(pathO2table); + gen->setVerbose(verbose); + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + gen->addHadronPDGs(std::stoi(spdg)); + printf("PDG %d \n", std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBJpsiDiMuon); + + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + // print debug + // gen->PrintDebug(); + + return gen; +} +FairGenerator* + GeneratorBeautyToPsiAndJpsi_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = -2.3, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +{ + auto gen = new o2::eventgen::GeneratorEvtGen(); + gen->setRapidity(rapidityMin, rapidityMax); + gen->setPDG(5); + gen->setRapidityHadron(rapidityMin,rapidityMax); + gen->setHadronMultiplicity(1); + TString pathO2table = gSystem->ExpandPathName("$O2DPG_ROOT/MC/config/PWGDQ/pythia8/decayer/switchOffBhadrons.cfg"); + gen->readFile(pathO2table.Data()); + gen->setConfigMBdecays(pathO2table); + gen->setVerbose(verbose); + + std::string spdg; + TObjArray* obj = pdgs.Tokenize(";"); + gen->SetSizePdg(obj->GetEntriesFast()); + for (int i = 0; i < obj->GetEntriesFast(); i++) { + spdg = obj->At(i)->GetName(); + gen->AddPdg(std::stoi(spdg), i); + printf("PDG %d \n", std::stoi(spdg)); + gen->addHadronPDGs(std::stoi(spdg)); + } + gen->SetForceDecay(kEvtBPsiAndJpsiDiMuon); + + // set random seed + gen->readString("Random:setSeed on"); + uint random_seed; + unsigned long long int random_value = 0; + ifstream urandom("/dev/urandom", ios::in|ios::binary); + urandom.read(reinterpret_cast(&random_value), sizeof(random_seed)); + gen->readString(Form("Random:seed = %d", random_value % 900000001)); + + // print debug + // gen->PrintDebug(); + + return gen; +} + diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini new file mode 100644 index 000000000..7ca57246a --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini @@ -0,0 +1,15 @@ +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_NonPromptSignals_gaptriggered_dq.C +funcName = GeneratorBeautyToPsiAndJpsi_EvtGenFwdY() + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-4.3,-2.3) + diff --git a/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini b/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini new file mode 100755 index 000000000..927428214 --- /dev/null +++ b/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C +funcName=GeneratorPythia8InjectedPromptCharmoniaGapTriggered(5,3) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg diff --git a/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini b/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini new file mode 100755 index 000000000..79d17674c --- /dev/null +++ b/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C +funcName=GeneratorPythia8InjectedPromptCharmoniaGapTriggered(5,4) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg diff --git a/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini b/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini new file mode 100755 index 000000000..2d62ec371 --- /dev/null +++ b/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/generator_pythia8_withInjectedPromptSignals_gaptriggered_dq.C +funcName=GeneratorPythia8InjectedPromptCharmoniaGapTriggered(5,5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGDQ/pythia8/generator/pythia8_inel_triggerGap.cfg diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.C new file mode 100644 index 000000000..d68eb1bcf --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.C @@ -0,0 +1,93 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 13; + double rapiditymin = -4.3; double rapiditymax = -2.3; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Int_t bpdgs[] = {511, 521, 531, 5112, 5122, 5232, 5132}; + Int_t sizePdg = sizeof(bpdgs)/sizeof(Int_t); + Bool_t hasBeautyMoth = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + // check if mothers are beauty hadrons + hasBeautyMoth = kFALSE; + if(idMoth){ // check beauty mother + auto tdM = mcreader.getTrack(i, idMoth); + for(int i=0; iGetPdgCode()) == bpdgs[i] ) hasBeautyMoth = kTRUE; } + } + if(hasBeautyMoth){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(rapidity > rapiditymin && rapidity < rapiditymax) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (jpsi <- b): " << nSignalJpsi << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalJpsiWithinAcc << "\n" + << "#signal (psi2S <- b): " << nSignalPsi2S << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.C b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.C new file mode 100644 index 000000000..e88864af3 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.C @@ -0,0 +1,85 @@ +int External() +{ + int checkPdgSignal[] = {443,100443}; + int checkPdgDecay = 13; + double rapiditymin = -4.3; double rapiditymax = -2.3; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalPsi2S{}; + int nSignalJpsiWithinAcc{}; + int nSignalPsi2SWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Bool_t isInjected = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0] || pdg == checkPdgSignal[1]) { + if(idMoth < 0){ + // count signal PDG + pdg == checkPdgSignal[0] ? nSignalJpsi++ : nSignalPsi2S++; + // count signal PDG within acceptance + if(rapidity > rapiditymin && rapidity < rapiditymax) { pdg == checkPdgSignal[0] ? nSignalJpsiWithinAcc++ : nSignalPsi2SWithinAcc++;} + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (prompt Jpsi): " << nSignalJpsi << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalJpsiWithinAcc << "\n" + << "#signal (prompt Psi(2S)): " << nSignalPsi2S << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalPsi2SWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptJpsiFwdy_TriggerGap.C b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptJpsiFwdy_TriggerGap.C new file mode 100644 index 000000000..2b75fcdbd --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptJpsiFwdy_TriggerGap.C @@ -0,0 +1,82 @@ +int External() +{ + int checkPdgSignal[] = {443}; + int checkPdgDecay = 13; + double rapiditymin = -4.3; double rapiditymax = -2.3; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal[0] << "\n decay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalJpsiWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Bool_t isInjected = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0]) { + if(idMoth < 0){ + // count signal PDG + nSignalJpsi++; + // count signal PDG within acceptance + if(rapidity > rapiditymin && rapidity < rapiditymax) nSignalJpsiWithinAcc++; + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (prompt Jpsi): " << nSignalJpsi << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalJpsiWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptPsi2SFwdy_TriggerGap.C b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptPsi2SFwdy_TriggerGap.C new file mode 100644 index 000000000..68477b621 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/Generator_InjectedPromptPsi2SFwdy_TriggerGap.C @@ -0,0 +1,82 @@ +int External() +{ + int checkPdgSignal[] = {100443}; + int checkPdgDecay = 13; + double rapiditymin = -4.3; double rapiditymax = -2.3; + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal[0] << "\n decay PDG " << checkPdgDecay << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nAntileptons{}; + int nLeptonPairs{}; + int nLeptonPairsToBeDone{}; + int nSignalJpsi{}; + int nSignalJpsiWithinAcc{}; + auto nEvents = tree->GetEntries(); + o2::steer::MCKinematicsReader mcreader("o2sim", o2::steer::MCKinematicsReader::Mode::kMCKine); + Bool_t isInjected = kFALSE; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto rapidity = track.GetRapidity(); + auto idMoth = track.getMotherTrackId(); + if (pdg == checkPdgDecay) { + // count leptons + nLeptons++; + } else if(pdg == -checkPdgDecay) { + // count anti-leptons + nAntileptons++; + } else if (pdg == checkPdgSignal[0]) { + if(idMoth < 0){ + // count signal PDG + nSignalJpsi++; + // count signal PDG within acceptance + if(rapidity > rapiditymin && rapidity < rapiditymax) nSignalJpsiWithinAcc++; + } + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) { + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << checkPdgSignal << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if (std::abs(pdg0) == checkPdgDecay && std::abs(pdg1) == checkPdgDecay && pdg0 == -pdg1) { + nLeptonPairs++; + if (child0->getToBeDone() && child1->getToBeDone()) { + nLeptonPairsToBeDone++; + } + } + } + } + } + } + std::cout << "#events: " << nEvents << "\n" + << "#leptons: " << nLeptons << "\n" + << "#antileptons: " << nAntileptons << "\n" + << "#signal (prompt Psi2S): " << nSignalJpsi << "; within acceptance " << rapiditymin << " < y < " << rapiditymax << " : " << nSignalJpsiWithinAcc << "\n" + << "#lepton pairs: " << nLeptonPairs << "\n" + << "#lepton pairs to be done: " << nLeptonPairs << "\n"; + + + if (nLeptonPairs == 0 || nLeptons == 0 || nAntileptons == 0) { + std::cerr << "Number of leptons, number of anti-leptons as well as number of lepton pairs should all be greater than 1.\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of lepton pairs should be the same as the number of lepton pairs which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh new file mode 100755 index 000000000..90f4fac9a --- /dev/null +++ b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh index 145c29d6f..4b0f86376 100755 --- a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh +++ b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh @@ -14,7 +14,7 @@ NBKGEVENTS=${NBKGEVENTS:-1} NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} -${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini --mft-reco-full # run workflow diff --git a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh new file mode 100755 index 000000000..fc637c33d --- /dev/null +++ b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh new file mode 100755 index 000000000..d1f03343f --- /dev/null +++ b/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh new file mode 100755 index 000000000..406152d24 --- /dev/null +++ b/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json From 76569d1b5d6806ada88c8062fe09f5263d3086ef Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 23 Oct 2023 13:07:53 +0200 Subject: [PATCH 0138/1239] If ZDC is not there, scale with FT0 --- .../configurations/asyncReco/setenv_extra.sh | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 0efa19765..debad920a 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -206,13 +206,24 @@ if [[ -z $RUN_IR ]] || [[ -z $RUN_DURATION ]] || [[ -z $RUN_BFIELD ]]; then export RUN_IR=`cat IR.txt` export RUN_DURATION=`cat Duration.txt` export RUN_BFIELD=`cat BField.txt` + export RUN_DETECTOR_LIST=`cat DetList.txt` fi +echo "DETECTOR LIST for current run ($RUNNUMBER) = $RUN_DETECTOR_LIST" +echo "DURATION for current run ($RUNNUMBER) = $RUN_DURATION" +echo "B FIELD for current run ($RUNNUMBER) = $RUN_BFIELD" echo "IR for current run ($RUNNUMBER) = $RUN_IR" if (( $(echo "$RUN_IR <= 0" | bc -l) )); then echo "Changing run IR to 1 Hz, because $RUN_IR makes no sense" RUN_IR=1 fi -echo "Duration of current run ($RUNNUMBER) = $RUN_DURATION" + +# Let's check if ZDC is in the detector list; this is needed for TPC dist correction scaling in PbPb 2023 +SCALE_WITH_ZDC=1 +SCALE_WITH_FT0=1 +isZDCinDataTaking=`echo $RUN_DETECTOR_LIST | grep ZDC` +isFT0inDataTaking=`echo $RUN_DETECTOR_LIST | grep FT0` +[[ -z $isZDCinDataTaking ]] && SCALE_WITH_ZDC=0 +[[ -z $isFT0inDataTaking ]] && SCALE_WITH_FT0=0 # For runs shorter than 10 minutes we have only a single slot. # In that case we have to adopt the slot length in order to @@ -315,7 +326,16 @@ elif [[ $ALIGNLEVEL == 1 ]]; then if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]]; then unset TPC_CORR_SCALING - export TPC_CORR_SCALING="--ctp-lumi-factor 2.414 --require-ctp-lumi" + export TPC_CORR_SCALING=" --ctp-lumi-factor 2.414 --require-ctp-lumi" + if [[ $SCALE_WITH_ZDC == 0 ]]; then + # scaling with FT0 + if [[ $SCALE_WITH_FT0 == 1 ]]; then + export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --require-ctp-lumi ' + else + echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." + return 1 + fi + fi fi if [[ $PERIOD != @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]] ; then From 466c771a08fe61371ca0dcaa0677c3b1ba8ad2eb Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 24 Oct 2023 15:34:27 +0200 Subject: [PATCH 0139/1239] TRD Kr workflow bug fixes (#1295) --- DATA/testing/detectors/TRD/trd-krypton.sh | 4 ++-- DATA/testing/detectors/TRD/workflows.desc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 DATA/testing/detectors/TRD/trd-krypton.sh diff --git a/DATA/testing/detectors/TRD/trd-krypton.sh b/DATA/testing/detectors/TRD/trd-krypton.sh old mode 100644 new mode 100755 index fd786f116..cfeec45d8 --- a/DATA/testing/detectors/TRD/trd-krypton.sh +++ b/DATA/testing/detectors/TRD/trd-krypton.sh @@ -35,13 +35,13 @@ CTF_CONFIG+=" --require-free-disk 53687091200 --wait-for-free-disk $CTF_FREE_DIS WORKFLOW= add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 add_W o2-trd-datareader "--disable-root-output --pipeline trd-datareader:$TRD_N_READERS" -add_W o2-trd-kr-clusterer "--meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --autosave-interval 105000 --pipeline trd-kr-clusterer:8" +add_W o2-trd-kr-clusterer "--disable-root-input --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --autosave-interval 105000 --pipeline trd-kr-clusterer:8" if workflow_has_parameter QC && has_detector_qc TRD; then add_QC_from_consul "/o2/components/qc/ANY/any/trd-full-qcmn-test" "--local --host epn -b" fi if workflow_has_parameter CTF; then add_W o2-trd-entropy-encoder-workflow "$RANS_OPT --mem-factor ${TRD_ENC_MEMFACT:-1.5} --pipeline trd-entropy-encoder:$TRD_N_ENCODER" - add_W o2-ctf-writer-workflow "$CONFIG_CTF" + add_W o2-ctf-writer-workflow "$CTF_CONFIG" fi # Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) diff --git a/DATA/testing/detectors/TRD/workflows.desc b/DATA/testing/detectors/TRD/workflows.desc index b0624af62..216b2df8e 100644 --- a/DATA/testing/detectors/TRD/workflows.desc +++ b/DATA/testing/detectors/TRD/workflows.desc @@ -1,2 +1,2 @@ TRD-noise: "O2PDPSuite" reco,1,1,"SHMSIZE=$((128 << 30)) testing/detectors/TRD/trd-noise.sh" -TRD-krypton: "O2PDPSuite" reco,40,35,"SHMSIZE=$((128 << 30)) testing/detectors/TRD/trd-kryptona.sh" +TRD-krypton: "O2PDPSuite" reco,40,35,"SHMSIZE=$((128 << 30)) testing/detectors/TRD/trd-krypton.sh" From 8a6b395484431c3c0ade1a355962b546e11c850a Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 24 Oct 2023 15:51:20 +0200 Subject: [PATCH 0140/1239] To enable DPL REPORT PROCESSING from JDL --- DATA/production/configurations/asyncReco/async_pass.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index fce992d80..66fd40566 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -287,6 +287,12 @@ else INPUT_TYPE=CTF fi +# enabling time reporting +if [[ -n $ALIEN_JDL_DPLREPORTPROCESSING ]]; then + export DPL_REPORT_PROCESSING=$ALIEN_JDL_DPLREPORTPROCESSING +fi + +# defining whether to keep files if [[ -n $ALIEN_JDL_PACKAGES ]]; then # if we have this env variable, it means that we are running on the grid # JDL can set the permille to keep; otherwise we use 2 if [[ ! -z "$ALIEN_JDL_NKEEP" ]]; then export NKEEP=$ALIEN_JDL_NKEEP; else NKEEP=2; fi From 3dd77adf533dbd86aca6929def29fe3ede0d290b Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 24 Oct 2023 16:13:54 +0200 Subject: [PATCH 0141/1239] typo fixed --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index debad920a..ebe3c4a2a 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -330,7 +330,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then if [[ $SCALE_WITH_ZDC == 0 ]]; then # scaling with FT0 if [[ $SCALE_WITH_FT0 == 1 ]]; then - export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --require-ctp-lumi ' + export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --require-ctp-lumi " else echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." return 1 From 0e81e65098b3db5f27eb1480fa1f7e73dd378227 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Tue, 24 Oct 2023 20:43:44 +0200 Subject: [PATCH 0142/1239] Tpc update qc (#1270) * Update tpc.json * Update tpc.json * Update tpc.json * Update tpc.json --------- Co-authored-by: David Rohr --- DATA/production/qc-async/tpc.json | 18 ++++++++++++++++-- DATA/production/qc-sync/tpc.json | 18 ++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index af807e718..10351d4ed 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -84,8 +84,8 @@ "resetAfterCycles": "1", "maxNumberCycles": "-1", "dataSource": { - "type": "direct", - "query" : "inputTracks:TPC/TRACKS/0" + "type": "dataSamplingPolicy", + "name": "tpc-tracks" }, "taskParameters": { "cutAbsEta": "1.", @@ -120,6 +120,20 @@ } ], "blocking": "false" + }, + { + "id": "tpc-tracks", + "active": "true", + "machines": [], + "query" : "inputTracks:TPC/TRACKS/0", + "samplingConditions": [ + { + "condition": "random", + "fraction": "0.1", + "seed": "1234" + } + ], + "blocking": "false" } ] } diff --git a/DATA/production/qc-sync/tpc.json b/DATA/production/qc-sync/tpc.json index c97687b4c..d11c1581a 100644 --- a/DATA/production/qc-sync/tpc.json +++ b/DATA/production/qc-sync/tpc.json @@ -108,8 +108,8 @@ "resetAfterCycles": "10", "maxNumberCycles": "-1", "dataSource": { - "type": "direct", - "query": "inputTracks:TPC/TRACKS/0" + "type": "dataSamplingPolicy", + "name": "tpc-tracks" }, "taskParameters": { "cutAbsEta": "1.", @@ -147,6 +147,20 @@ } ], "blocking": "false" + }, + { + "id": "tpc-tracks", + "active": "true", + "machines": [], + "query" : "inputTracks:TPC/TRACKS/0", + "samplingConditions": [ + { + "condition": "random", + "fraction": "0.01", + "seed": "1234" + } + ], + "blocking": "false" } ] } From 9820c2b93048ad24b0155df7ab0c3293077fe638 Mon Sep 17 00:00:00 2001 From: iravasen Date: Thu, 14 Sep 2023 15:45:05 +0200 Subject: [PATCH 0143/1239] Update its-noise-processing.sh --- DATA/production/calib/its-noise-processing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/calib/its-noise-processing.sh b/DATA/production/calib/its-noise-processing.sh index ee83fb3d6..3d0181a14 100755 --- a/DATA/production/calib/its-noise-processing.sh +++ b/DATA/production/calib/its-noise-processing.sh @@ -21,7 +21,7 @@ fi WORKFLOW= add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1\"" "" 0 -add_W o2-itsmft-stf-decoder-workflow "${OUTTYPE} --nthreads ${NITSDECTHREADS} --pipeline its-stf-decoder:${NITSDECTPIPELINES}" +add_W o2-itsmft-stf-decoder-workflow "${OUTTYPE} --nthreads ${NITSDECTHREADS} --pipeline its-stf-decoder:${NITSDECTPIPELINES} --ignore-noise-map" add_W o2-dpl-output-proxy "--dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-noise-input-proxy --channel-config \"name=its-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=1\"" "" 0 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" From 5c66ba52ff114ab2565bea7615e4c0dd90848bed Mon Sep 17 00:00:00 2001 From: iravasen Date: Fri, 15 Sep 2023 10:34:47 +0200 Subject: [PATCH 0144/1239] added a noise calib workflow for tests: push to ccdb-test --- DATA/production/calib/its-noise-aggregator.sh | 2 +- DATA/production/standalone-calibration.desc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DATA/production/calib/its-noise-aggregator.sh b/DATA/production/calib/its-noise-aggregator.sh index d0636f0d9..97bfcf696 100755 --- a/DATA/production/calib/its-noise-aggregator.sh +++ b/DATA/production/calib/its-noise-aggregator.sh @@ -18,7 +18,7 @@ if [[ -z $NTHREADS ]] ; then NTHREADS=1; fi CCDBPATH1="http://o2-ccdb.internal" CCDBPATH2="http://alio2-cr1-flp199.cern.ch:8083" -if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" || ! -z $ISTEST ]]; then CCDBPATH1="http://ccdb-test.cern.ch:8080" CCDBPATH2="http://ccdb-test.cern.ch:8080" fi diff --git a/DATA/production/standalone-calibration.desc b/DATA/production/standalone-calibration.desc index 9d4d66b3d..306cdc855 100644 --- a/DATA/production/standalone-calibration.desc +++ b/DATA/production/standalone-calibration.desc @@ -2,6 +2,8 @@ FT0-time-offset-calibration: "O2PDPSuite" reco,5,5,"production/calib/ft0-timeoff ITS-noise-calibration: "O2PDPSuite" reco,20,20,"NITSDECTHREADS=4 NITSDECTPIPELINES=6 production/calib/its-noise-processing.sh" calib,20,"NTHREADSACC=4 NTHREADSNORM=16 NITSACCPIPELINES=16 production/calib/its-noise-aggregator.sh" +ITS-noise-calibration-test: "O2PDPSuite" reco,20,20,"NITSDECTHREADS=4 NITSDECTPIPELINES=6 production/calib/its-noise-processing.sh" calib,20,"NTHREADSACC=4 NTHREADSNORM=16 NITSACCPIPELINES=16 ISTEST=1 production/calib/its-noise-aggregator.sh" + ITS-noise-calibration-clusters: "O2PDPSuite" reco,20,20,"NITSDECTHREADS=4 NITSDECTPIPELINES=6 USECLUSTERS=1 production/calib/its-noise-processing.sh" calib,20,"USECLUSTERS=1 NTHREADS=32 production/calib/its-noise-aggregator.sh" ITS-thr-tuning: "O2PDPSuite" reco,40,40,"RUNTYPE_ITS=tuning production/calib/its-threshold-processing.sh" calib,40,"RUNTYPE_ITS=tuning production/calib/its-threshold-aggregator.sh" From 6a8cb0c0867cdb35a1e657d62ab9b37f96ac3800 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 24 Oct 2023 23:06:11 +0200 Subject: [PATCH 0145/1239] Possibility to limit the number of CTF files for debug --- .../configurations/asyncReco/async_pass.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 66fd40566..5e7ff77a4 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -48,6 +48,18 @@ if [[ -f list.list ]]; then echo -e "\n" fi +# Could need sometimes to iterate just a subset of the input files +# +[ -z ${ALIEN_JDL_INPUTFILELIMIT} ] && ALIEN_JDL_INPUTFILELIMIT=($(cat list.list|wc -l)) +head -${ALIEN_JDL_INPUTFILELIMIT} list.list > list.listtmp && mv list.listtmp list.list +echo "Will iterate ${ALIEN_JDL_INPUTFILELIMIT} input files" + +if [[ -f list.list ]]; then + echo "Processing will be on the following list of files:" + cat list.list + echo -e "\n" +fi + POSITIONAL=() while [[ $# -gt 0 ]]; do key="$1" From e0e805de1e206a9c74082c01a751d910458301cb Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 24 Oct 2023 23:20:27 +0200 Subject: [PATCH 0146/1239] Avoid to print 2x --- DATA/production/configurations/asyncReco/async_pass.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 5e7ff77a4..8e879a06d 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -42,12 +42,6 @@ elif [[ $1 != "list.list" && "${1##*.}" == "list" ]]; then shift fi -if [[ -f list.list ]]; then - echo "Processing will be on the following list of files:" - cat list.list - echo -e "\n" -fi - # Could need sometimes to iterate just a subset of the input files # [ -z ${ALIEN_JDL_INPUTFILELIMIT} ] && ALIEN_JDL_INPUTFILELIMIT=($(cat list.list|wc -l)) From aa54ffacb1537052808ea1368b75417f8df14c4e Mon Sep 17 00:00:00 2001 From: sevdokim Date: Tue, 24 Oct 2023 17:44:32 +0200 Subject: [PATCH 0147/1239] CPV, PHS: add QC tasks to MC --- MC/bin/o2dpg_qc_finalization_workflow.py | 3 + MC/bin/o2dpg_sim_workflow.py | 18 + MC/config/QC/json/cpv-clusters-task.json | 48 +++ MC/config/QC/json/cpv-digits-task.json | 48 +++ .../QC/json/phs-cells-clusters-task.json | 378 ++++++++++++++++++ MC/run/examples/README.md | 2 +- 6 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 MC/config/QC/json/cpv-clusters-task.json create mode 100644 MC/config/QC/json/cpv-digits-task.json create mode 100644 MC/config/QC/json/phs-cells-clusters-task.json diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index d0b70f7c4..09d1e7b24 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -98,6 +98,9 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif else: add_QC_finalization('tofPIDQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/pidtofNoTRD.json') add_QC_finalization('RecPointsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/ft0-reconstruction-config.json') + add_QC_finalization('CPVDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/cpv-digits-task.json') + add_QC_finalization('CPVClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/cpv-clusters-task.json') + add_QC_finalization('PHSCellsClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/phs-cells-clusters-task.json') # The list of QC Post-processing workflows add_QC_postprocessing('tofTrendingHits', 'json://${O2DPG_ROOT}/MC/config/QC/json/tof-trending-hits.json', [QC_finalize_name('tofDigitsQC')], runSpecific=False, prodSpecific=True) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index aa3fb5c26..b6c7e0152 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1272,6 +1272,24 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): readerCommand='o2-global-track-cluster-reader --track-types "ITS" --cluster-types "ITS"', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/its-clusters-tracks-qc.json') + ### CPV + if isActive('CPV'): + addQCPerTF(taskName='CPVDigitsQC', + needs=[getDigiTaskName("CPV")], + readerCommand='o2-cpv-digit-reader-workflow', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/cpv-digits-task.json') + addQCPerTF(taskName='CPVClustersQC', + needs=[CPVRECOtask['name']], + readerCommand='o2-cpv-cluster-reader-workflow', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/cpv-clusters-task.json') + + ### PHS + if isActive('PHS'): + addQCPerTF(taskName='PHSCellsClustersQC', + needs=[PHSRECOtask['name']], + readerCommand='o2-phos-reco-workflow --input-type cells --output-type clusters --disable-mc --disable-root-output', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/phs-cells-clusters-task.json') + #secondary vertexer svfinder_threads = ' --threads 1 ' svfinder_cpu = 1 diff --git a/MC/config/QC/json/cpv-clusters-task.json b/MC/config/QC/json/cpv-clusters-task.json new file mode 100644 index 000000000..6292682ac --- /dev/null +++ b/MC/config/QC/json/cpv-clusters-task.json @@ -0,0 +1,48 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "ccdb-test.cern.ch:8080" + } + }, + "tasks": { + "Clusters": { + "active": "true", + "className": "o2::quality_control_modules::cpv::PhysicsTask", + "moduleName": "QcCPV", + "detectorName": "CPV", + "cycleDurationSeconds": "30", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "clusters:CPV/CLUSTERS/0;ctrigrec:CPV/CLUSTERTRIGRECS/0" + }, + "taskParameters": { + "ccdbCheckInterval": "1000", + "isAsyncMode": "1" + } + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/cpv-digits-task.json b/MC/config/QC/json/cpv-digits-task.json new file mode 100644 index 000000000..bf6d788ef --- /dev/null +++ b/MC/config/QC/json/cpv-digits-task.json @@ -0,0 +1,48 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "ccdb-test.cern.ch:8080" + } + }, + "tasks": { + "Digits": { + "active": "true", + "className": "o2::quality_control_modules::cpv::PhysicsTask", + "moduleName": "QcCPV", + "detectorName": "CPV", + "cycleDurationSeconds": "30", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "digits:CPV/DIGITS/0;dtrigrec:CPV/DIGITTRIGREC/0" + }, + "taskParameters": { + "ccdbCheckInterval": "1000", + "isAsyncMode": "1" + } + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/phs-cells-clusters-task.json b/MC/config/QC/json/phs-cells-clusters-task.json new file mode 100644 index 000000000..bb56a398e --- /dev/null +++ b/MC/config/QC/json/phs-cells-clusters-task.json @@ -0,0 +1,378 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "43", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "infologger:///debug?qc" + }, + "consul": { + "url": "" + }, + "#conditionDB": { + "url": "ccdb-test.cern.ch:8080" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "Cells": { + "active": "true", + "className": "o2::quality_control_modules::phos::RawQcTask", + "moduleName": "QcPHOS", + "detectorName": "PHS", + "cycleDurationSeconds": "30", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "cells:PHS/CELLS/0;cellstr:PHS/CELLTRIGREC/0" + }, + "taskParameters": { + "physics": "on" + } + }, + "Clusters": { + "active": "true", + "className": "o2::quality_control_modules::phos::ClusterQcTask", + "moduleName": "QcPHOS", + "detectorName": "PHS", + "cycleDurationSeconds": "30", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "clusters:PHS/CLUSTERS;clustertr:PHS/CLUSTERTRIGREC" + }, + "taskParameters": { + "mEnergyMinForInvMass": "0.3", + "mMultiplicityMinForInvMass": "2" + } + } + }, + "#checks": { + "CellsIncrease": { + "active": "false", + "className": "o2::quality_control_modules::common::IncreasingEntries", + "moduleName": "QcCommon", + "policy": "OnAny", + "detectorName": "PHS", + "checkParameters": { + "mustIncrease": "true" + }, + "dataSource": [ + { + "type": "Task", + "name": "RawTask", + "MOs": [ + "CellHGOccupancyM1", + "CellHGOccupancyM2", + "CellHGOccupancyM3", + "CellHGOccupancyM4" + ] + } + ] + }, + "ClustersIncrease": { + "active": "false", + "className": "o2::quality_control_modules::common::IncreasingEntries", + "moduleName": "QcCommon", + "policy": "OnAny", + "detectorName": "PHS", + "checkParameters": { + "mustIncrease": "true" + }, + "dataSource": [ + { + "type": "Task", + "name": "ClusterTask", + "MOs": [ + "SpectrumM1", + "SpectrumM2", + "SpectrumM3", + "SpectrumM4" + ] + } + ] + }, + "ErrorsCheck": { + "active": "false", + "className": "o2::quality_control_modules::phos::RawCheck", + "moduleName": "QcPHOS", + "policy": "OnAny", + "detectorName": "PHS", + "checkParameters": { + "mErrorOccuranceThreshold0": "20.1", + "mErrorOccuranceThreshold1": "20.1", + "mErrorOccuranceThreshold2": "20.1", + "mErrorOccuranceThreshold3": "20.1", + "mErrorOccuranceThreshold4": "20.1" + }, + "dataSource": [ + { + "type": "Task", + "name": "RawTask", + "MOs": [ + "ErrorTypeOccurance" + ] + } + ] + }, + "CellsCheck": { + "active": "false", + "className": "o2::quality_control_modules::phos::RawCheck", + "moduleName": "QcPHOS", + "policy": "OnAny", + "detectorName": "PHS", + "checkParameters": { + "mToleratedBadChannelsM1": "1", + "mToleratedBadChannelsM2": "1", + "mToleratedBadChannelsM3": "1", + "mToleratedBadChannelsM4": "1" + }, + "dataSource": [ + { + "type": "Task", + "name": "RawTask", + "MOs": [ + "BadMapSummary", + "CellHGOccupancyM1", + "CellHGOccupancyM2", + "CellHGOccupancyM3", + "CellHGOccupancyM4" + ] + } + ] + }, + "ClustersCheck": { + "active": "false", + "className": "o2::quality_control_modules::phos::ClusterCheck", + "moduleName": "QcPHOS", + "policy": "OnAny", + "detectorName": "PHS", + "checkParameters": { + "mMaxCluEnergyMean2": "1.0", + "mMinCluEnergyMean1": "9.0" + }, + "dataSource": [ + { + "type": "Task", + "name": "ClusterTask", + "MOs": [ + "SpectrumM1", + "SpectrumM2", + "SpectrumM3", + "SpectrumM4" + ] + } + ] + } + }, + "#postprocessing": { + "PhysicsTrending": { + "active": "true", + "className": "o2::quality_control::postprocessing::SliceTrendingTask", + "moduleName": "QualityControl", + "detectorName": "PHS", + "resumeTrend": "false", + "producePlotsOnUpdate": "true", + "initTrigger": [ + "once" + ], + "updateTrigger": [ + "newobject:qcdb:PHS/MO/ClusterTask/SpectrumM1" + ], + "stopTrigger": [ + "usercontrol" + ], + "dataSources": [ + { + "type": "repository", + "path": "PHS/MO/ClusterTask", + "names": [ + "SpectrumM1", + "SpectrumM2", + "SpectrumM3", + "SpectrumM4" + ], + "reductorName": "o2::quality_control_modules::common::TH1SliceReductor", + "axisDivision": [ + [ + "1.", + "10." + ] + ], + "moduleName": "QcCommon" + } + ], + "plots": [ + { + "name": "mean_of_cluEnergyM1", + "title": "Trend of mean energy of >1GeV clusters", + "varexp": "SpectrumM1.meanX:time", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.", + "graphAxisLabel": "Mean energy:time", + "graphYRange": "0:10" + }, + { + "name": "mean_of_cluEnergyM2", + "title": "Trend of mean energy of >1GeV clusters", + "varexp": "SpectrumM2.meanX:time", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.", + "graphAxisLabel": "Mean energy:time", + "graphYRange": "0:10" + }, + { + "name": "mean_of_cluEnergyM3", + "title": "Trend of mean energy of >1GeV clusters", + "varexp": "SpectrumM3.meanX:time", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.", + "graphAxisLabel": "Mean energy:time", + "graphYRange": "0:10" + }, + { + "name": "mean_of_cluEnergyM4", + "title": "Trend of mean energy of >1GeV clusters", + "varexp": "SpectrumM4.meanX:time", + "selection": "", + "option": "*L", + "graphErrors": "errMeanX:0.", + "graphAxisLabel": "Mean energy:time", + "graphYRange": "0:10" + } + ] + }, + "QualityTask": { + "active": "true", + "className": "o2::quality_control_modules::common::QualityTask", + "moduleName": "QcCommon", + "detectorName": "PHS", + "qualityGroups": [ + { + "name": "global", + "title": "GLOBAL PHS QUALITY", + "path": "PHS/QO/GlobalQuality", + "ignoreQualitiesDetails": [ + "Null", + "Good", + "Medium", + "Bad" + ], + "inputObjects": [ + { + "name": "GlobalQuality", + "title": "Global PHS Quality", + "messageBad": "Inform PHS on-call", + "messageMedium": "Inform the PHS on-call", + "messageGood": "All checks are OK", + "messageNull": "Some histograms are empty!!!" + } + ] + }, + { + "name": "details", + "title": "PHS DETAILS", + "path": "PHS/QO", + "ignoreQualitiesDetails": [], + "inputObjects": [ + { + "name": "CellsIncrease", + "title": "Number of cells increases", + "messageBad": "Entries are not increasing in last cycle", + "messageNull": "" + }, + { + "name": "ClustersIncrease", + "title": "Number of clusters increases", + "messageBad": "Entries are not increasing in last cycle", + "messageNull": "" + }, + { + "name": "CellsCheck", + "title": "Cells check", + "messageBad": "Inform PHS on-call", + "messageMedium": "Inform the PHS on-call", + "messageGood": "", + "messageNull": "" + }, + { + "name": "ClustersCheck", + "title": "Clusters check", + "messageBad": "Inform PHS on-call", + "messageMedium": "Inform the PHS on-call", + "messageGood": "", + "messageNull": "Not enough stat" + }, + { + "name": "ErrorsCheck", + "title": "Errors check", + "messageBad": "Inform PHS on-call", + "messageMedium": "Inform the PHS on-call", + "messageGood": "", + "messageNull": "" + } + ] + } + ], + "initTrigger": [ + "newobject:qcdb:PHS/QO/GlobalQuality/GlobalQuality" + ], + "updateTrigger": [ + "newobject:qcdb:PHS/QO/GlobalQuality/GlobalQuality" + ], + "stopTrigger": [ + "userorcontrol" + ] + } + }, + "#aggregators": { + "GlobalQuality": { + "active": "true", + "className": "o2::quality_control_modules::common::WorstOfAllAggregator", + "moduleName": "QcCommon", + "policy": "OnAll", + "detectorName": "PHS", + "dataSource": [ + { + "type": "Check", + "name": "CellsIncrease" + }, + { + "type": "Check", + "name": "ClustersIncrease" + }, + { + "type": "Check", + "name": "CellsCheck" + }, + { + "type": "Check", + "name": "ClustersCheck" + }, + { + "type": "Check", + "name": "ErrorsCheck" + } + ] + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/run/examples/README.md b/MC/run/examples/README.md index 08e0d231f..0de36df94 100644 --- a/MC/run/examples/README.md +++ b/MC/run/examples/README.md @@ -10,7 +10,7 @@ The script [O2DPG_pp_minbias.sh](O2DPG_pp_minbias.sh) contains 4 steps: To execute the script (here with QC and test analyses), run ```bash -DOQC=1 DOANALYSIS=1 ${O2DPPG_ROOT}/MC/run/examples/O2DPG_pp_minbias.sh +DOQC=1 DOANALYSIS=1 ${O2DPG_ROOT}/MC/run/examples/O2DPG_pp_minbias.sh ``` There are additional settings to control memory can CPU usage. Similar to the `DOQC` or `DOANALYSIS` flag, you can prepend for instance From 81301532b7f653feadd4d2a2ca730b583b56832f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 25 Oct 2023 21:45:05 +0200 Subject: [PATCH 0148/1239] async workflow: tune multiplicities of 1NUMA workflow on EPN --- DATA/production/workflow-multiplicities.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index b3d4de384..21bf3b10e 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -100,20 +100,27 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=20 [[ -z $SHMSIZE ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4 + [[ -z $TIMEFRAME_SHM_LIMIT ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3)) NGPURECOTHREADS=8 NTRDTRKTHREADS=4 + ITSTRK_THREADS=6 + ITSTPC_THREADS=3 + SVERTEX_THREADS=40 + N_SECVTX=1 NGPUS=4 N_TPCTRK=4 # time in s: pvtx 16, tof 30, trd 82 itstpc 53 its 200 mfttr 30 tpcent 23 hmp-clus 40 (25.11.22) - N_TPCENTDEC=$(math_max $((2 * $NGPUS / 4)) 1) + N_TPCENTDEC=$(math_max $((5 * $NGPUS / 4)) 1) N_ITSTRK=$(math_max $((10 * $NGPUS / 4)) 1) - N_TPCITS=$(math_max $((4 * $NGPUS / 4)) 1) + N_TPCITS=$(math_max $((7 * $NGPUS / 4)) 1) N_MFTTRK=$(math_max $((3 * $NGPUS / 4)) 1) - N_TRDTRK=$(math_max $((7 * $NGPUS / 4)) 1) - N_TOFMATCH=$(math_max $((3 * $NGPUS / 4)) 1) + N_TRDTRK=$(math_max $((9 * $NGPUS / 4)) 1) + N_TOFMATCH=$(math_max $((11 * $NGPUS / 4)) 1) N_HMPCLUS=$(math_max $((3 * $NGPUS / 4)) 1) - CONFIG_EXTRA_PROCESS_o2_its_reco_workflow="ITSVertexerParam.nThreads=3;ITSCATrackerParam.nThreads=3;" - N_MCHCL=2 + N_AODPROD=2 + N_MCHCL=9 + N_HMPMATCH=14 + N_MCHTRK=7 N_PRIMVTXMATCH=2 N_PRIMVTX=3 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_64cpu" ]]; then From 98163be34d5514fcf37ad9524f1a47b26acf7d89 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Mon, 23 Oct 2023 13:53:04 +0200 Subject: [PATCH 0149/1239] Small fix to heavy-quark rapidity check --- .../generator/generator_pythia8_gaptriggered_hf.C | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 75c2663b1..f2e591fc4 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -88,12 +88,18 @@ protected: auto daughterList = event[iPart].daughterList(); bool hasQ = false, hasQbar = false, atSelectedY = false; for (auto iDau : daughterList) { - if (event[iDau].id() == mQuarkPdg) + if (event[iDau].id() == mQuarkPdg) { hasQ = true; - if (event[iDau].id() == -mQuarkPdg) + if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) { + atSelectedY = true; + } + } + if (event[iDau].id() == -mQuarkPdg) { hasQbar = true; - if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) - atSelectedY = true; + if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) { + atSelectedY = true; + } + } } if (hasQ && hasQbar && atSelectedY) { isGoodAtPartonLevel = true; From 73f82af5f8671a58e87095efd7458aa5888bb60c Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Thu, 19 Oct 2023 17:16:17 +0200 Subject: [PATCH 0150/1239] Add generator information to the event --- .../generator/generator_pythia8_gaptriggered_hf.C | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index f2e591fc4..c7418c048 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -30,6 +30,14 @@ public: /// Destructor ~GeneratorPythia8GapTriggeredHF() = default; + Bool_t Init() override + { + addSubGenerator(0, "Minimum bias"); + addSubGenerator(4, "Charm injected"); + addSubGenerator(5, "Beauty injected"); + return o2::eventgen::GeneratorPythia8::Init(); + } + void addTriggerOnHadron(int hadPdg) { mHadronPdg = hadPdg; }; void setQuarkTrigger (bool doNoQuarkTrigger) { mDoNoQuarkTrigger = doNoQuarkTrigger; }; void setQuarkRapidity(float yMin, float yMax) @@ -56,12 +64,14 @@ protected: genOk = selectEvent(mPythia.event); } } + notifySubGenerator(mQuarkPdg); } else { // Generate minimum-bias event bool genOk = false; while (!genOk) { genOk = GeneratorPythia8::generateEvent(); } + notifySubGenerator(0); } mGeneratedEvents++; From 4bc7acb43c5df69a4fe44456dba5dbc8ea3e5dc8 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Thu, 19 Oct 2023 17:41:05 +0200 Subject: [PATCH 0151/1239] Add alternate injection of charm and beauty --- .../generator_pythia8_gaptriggered_hf.C | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index c7418c048..877bcc016 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -25,6 +25,7 @@ public: mHadRapidityMin = -1.5; mHadRapidityMax = -1.5; mDoNoQuarkTrigger = false; + mDoAltInjection = false; } /// Destructor @@ -50,6 +51,7 @@ public: mHadRapidityMin = yMin; mHadRapidityMax = yMax; }; + void setAlternateInjection(bool doAltInjection) {mDoAltInjection = doAltInjection; }; protected: //__________________________________________________________________ @@ -65,6 +67,12 @@ protected: } } notifySubGenerator(mQuarkPdg); + + // Alternate charm and beauty if enabled (with the same ratio) + if(mDoAltInjection) { + mQuarkPdg = (mQuarkPdg == 4) ? 5 : 4; + } + } else { // Generate minimum-bias event bool genOk = false; @@ -150,10 +158,12 @@ private: // Control gap-triggering unsigned long long mGeneratedEvents; int mInverseTriggerRatio; + + // Control alternate injection of charm and beauty + bool mDoAltInjection; }; // Predefined generators: - // Charm-enriched FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 4); @@ -170,16 +180,32 @@ FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQ } // Beauty-enriched -FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float yMin=-1.5, float yMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { +FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 5); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); - myGen->setQuarkRapidity(yMin, yMax); + myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); + if(pdgCodeCharmHadron != 0) { + myGen->setQuarkTrigger(doNoQuarkTrigger); + myGen->addTriggerOnHadron(pdgCodeCharmHadron); + myGen->setHadronRapidity(yHadronMin, yHadronMax); + } + return myGen; +} + +// Charm and beauty enriched (with same ratio) +FairGenerator *GeneratorPythia8GapTriggeredCharmAndBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 4); + auto seed = (gRandom->TRandom::GetSeed() % 900000000); + myGen->readString("Random:setSeed on"); + myGen->readString("Random:seed " + std::to_string(seed)); + myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); if(pdgCodeCharmHadron != 0) { myGen->setQuarkTrigger(doNoQuarkTrigger); myGen->addTriggerOnHadron(pdgCodeCharmHadron); myGen->setHadronRapidity(yHadronMin, yHadronMax); } + myGen->setAlternateInjection(true); return myGen; } From d94d88bf923099655027f5838e48315696ea6b62 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Wed, 25 Oct 2023 12:12:33 +0200 Subject: [PATCH 0152/1239] Fix return type --- .../external/generator/generator_pythia8_gaptriggered_hf.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 877bcc016..2bc76fb9e 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -31,7 +31,7 @@ public: /// Destructor ~GeneratorPythia8GapTriggeredHF() = default; - Bool_t Init() override + bool Init() override { addSubGenerator(0, "Minimum bias"); addSubGenerator(4, "Charm injected"); From e37713111bbf3dc744d7b7594fe2723fcab1c532 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Wed, 25 Oct 2023 16:16:36 +0200 Subject: [PATCH 0153/1239] Improve handling of alternate injection --- .../generator_pythia8_gaptriggered_hf.C | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 2bc76fb9e..0bc08dfa5 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -4,6 +4,7 @@ #include "TRandom.h" #include +#include using namespace Pythia8; @@ -14,18 +15,19 @@ public: GeneratorPythia8GapTriggeredHF() = default; /// constructor - GeneratorPythia8GapTriggeredHF(int inputTriggerRatio = 5, int quarkPdg = 4) { + GeneratorPythia8GapTriggeredHF(int inputTriggerRatio = 5, std::vector quarkPdgList = {4}, bool doAltInjection = false) { mGeneratedEvents = 0; mHadronPdg = 0; // unless differently set, we do not trigger on specific hadron species - mQuarkPdg = quarkPdg; + mQuarkPdg = quarkPdgList[0]; mInverseTriggerRatio = inputTriggerRatio; mQuarkRapidityMin = -1.5; mQuarkRapidityMax = -1.5; mHadRapidityMin = -1.5; mHadRapidityMax = -1.5; mDoNoQuarkTrigger = false; - mDoAltInjection = false; + mDoAltInjection = doAltInjection; + mQuarkPdgList = quarkPdgList; } /// Destructor @@ -51,7 +53,6 @@ public: mHadRapidityMin = yMin; mHadRapidityMax = yMax; }; - void setAlternateInjection(bool doAltInjection) {mDoAltInjection = doAltInjection; }; protected: //__________________________________________________________________ @@ -70,7 +71,7 @@ protected: // Alternate charm and beauty if enabled (with the same ratio) if(mDoAltInjection) { - mQuarkPdg = (mQuarkPdg == 4) ? 5 : 4; + mQuarkPdg = (mQuarkPdg == mQuarkPdgList[0]) ? mQuarkPdgList[1] : mQuarkPdgList[0]; } } else { @@ -161,12 +162,13 @@ private: // Control alternate injection of charm and beauty bool mDoAltInjection; + std::vector mQuarkPdgList; }; // Predefined generators: // Charm-enriched FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 4); + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4}, false); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); @@ -181,7 +183,7 @@ FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQ // Beauty-enriched FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 5); + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{5}, false); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); @@ -196,7 +198,7 @@ FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float y // Charm and beauty enriched (with same ratio) FairGenerator *GeneratorPythia8GapTriggeredCharmAndBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, 4); + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4, 5}, true); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); @@ -206,6 +208,5 @@ FairGenerator *GeneratorPythia8GapTriggeredCharmAndBeauty(int inputTriggerRatio, myGen->addTriggerOnHadron(pdgCodeCharmHadron); myGen->setHadronRapidity(yHadronMin, yHadronMax); } - myGen->setAlternateInjection(true); return myGen; } From 769dc0f8bb070163017b9c8801bdbb47172275c8 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Wed, 25 Oct 2023 19:00:18 +0200 Subject: [PATCH 0154/1239] Add D2H generators (temporary configuration) --- MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini | 7 +++++++ MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini | 7 +++++++ MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini new file mode 100755 index 000000000..1f04a3ade --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredBeauty(3, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini new file mode 100755 index 000000000..31ec8e82d --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini new file mode 100755 index 000000000..c5e668f9f --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg From 9bbe0aa9d804c3f7a4d09af23db408b9ce28ff47 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Wed, 25 Oct 2023 22:18:12 +0200 Subject: [PATCH 0155/1239] Add test macros --- .../PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C | 112 +++++++++++++++ .../PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C | 112 +++++++++++++++ .../tests/GeneratorHF_D2H_ccbar_and_bbbar.C | 129 ++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C new file mode 100644 index 000000000..4aedceb78 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C @@ -0,0 +1,112 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuark{5}; + float ratioTrigger = 1./3; // one event triggered out of 3 + + std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}}}, // Ds+ + {443, {{-11, 11}}}, // Jpsi + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInj{}; + int nQuarks{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuark) { + nEventsInj++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuark) { + nQuarks++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuark) << nEventsInj << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuark) << nQuarks << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInj < nEvents * ratioTrigger * 0.95 || nEventsInj > nEvents * ratioTrigger * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuark << " different than expected\n"; + return 1; + } + + if (nQuarks < 2 * nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuark << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C new file mode 100644 index 000000000..5013f4fd3 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C @@ -0,0 +1,112 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuark{4}; + float ratioTrigger = 1./3; // one event triggered out of 3 + + std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}}}, // Ds+ + {443, {{-11, 11}}}, // Jpsi + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInj{}; + int nQuarks{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuark) { + nEventsInj++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuark) { + nQuarks++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuark) << nEventsInj << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuark) << nQuarks << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInj < nEvents * ratioTrigger * 0.95 || nEventsInj > nEvents * ratioTrigger * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuark << " different than expected\n"; + return 1; + } + + if (nQuarks < 2 * nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuark << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C new file mode 100644 index 000000000..68dd44347 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C @@ -0,0 +1,129 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./2; // one event triggered out of 2 + + std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}}}, // Ds+ + {443, {{-11, 11}}}, // Jpsi + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} From cd9aabaac856c73ea4121d135a6be52473a82fa3 Mon Sep 17 00:00:00 2001 From: shahoian Date: Wed, 25 Oct 2023 17:44:54 +0200 Subject: [PATCH 0156/1239] Suppress obsolete SVertexer options --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index ebe3c4a2a..ccc019af4 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -414,14 +414,8 @@ fi if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " fi -export SVTX="svertexer.checkV0Hypothesis=false;svertexer.checkCascadeHypothesis=false" -# strangeness tracking -export STRK="" export CONFIG_EXTRA_PROCESS_o2_primary_vertexing_workflow+=";$PVERTEXER;$VDRIFTPARAMOPTION;" -export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";$SVTX" -export CONFIG_EXTRA_PROCESS_o2_strangeness_tracking_workflow+=";$STRK" - export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATCH;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" [[ ! -z "${TPCITSTIMEBIAS}" ]] && export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";tpcitsMatch.globalTimeBiasMUS=$TPCITSTIMEBIAS;" From f9d4eee4eb23eafdeccbe1c7a3a4ccbc98247446 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:39:07 +0200 Subject: [PATCH 0157/1239] Cleanup krypton workflows (#1299) --- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 35 +----------- DATA/testing/detectors/TPC/tpc-krypton.sh | 54 +++++-------------- DATA/testing/detectors/TPC/workflows.desc | 1 + 3 files changed, 17 insertions(+), 73 deletions(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index 3644177fb..5e5c3ce79 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -2,47 +2,16 @@ source common/setenv.sh - -export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder - source common/getCommonArgs.sh -if [ $NUMAGPUIDS != 0 ]; then - ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" -fi -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;GPU_global.mutexMemReg=true;" - GPU_CONFIG+=" --environment \"ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}\"" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi +export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder -if [ $GPUTYPE != "CPU" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then - HOSTMEMSIZE=$(( 1 << 30 )) - fi -fi -if [ $HOSTMEMSIZE != "0" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" -fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" NLANES=36 SESSION="default" -PIPEADD="0" ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" HOST=localhost QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" @@ -67,4 +36,4 @@ o2-dpl-raw-proxy $ARGS_ALL \ --max-tf-per-file 8000 \ --time-bins-before 20 \ | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} | grep -v ERROR + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index c2c9b1b16..13fe859e3 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -2,62 +2,34 @@ source common/setenv.sh +source common/getCommonArgs.sh + +source common/gen_topo_helper_functions.sh + + export GLOBAL_SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder -source common/getCommonArgs.sh if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi -if [ $GPUTYPE != "CPU" ]; then - ARGS_ALL+=" --shm-mlock-segment-on-creation 1" -fi - -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then - export TIMESLICEOFFSET=0 - else - export TIMESLICEOFFSET=$NGPUS - fi - GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;GPU_global.mutexMemReg=true;" - GPU_CONFIG+=" --environment \"ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}\"" - export HSA_NO_SCRATCH_RECLAIM=1 - #export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 -else - GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" -fi - -if [ $GPUTYPE != "CPU" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then - HOSTMEMSIZE=$(( 1 << 30 )) - fi -fi -if [ $HOSTMEMSIZE != "0" ]; then - GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" -fi PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -WRITER_TYPE="none" -workflow_has_parameter ROOT && export WRITER_TYPE="EPN" NLANES=1 SESSION="default" -PIPEADD="0" + ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-full-qcmn-krypton" -if [ $WRITER_TYPE == "EPN" ]; then - KR_CONFIG="--writer-type ${WRITER_TYPE} --meta-output-dir /data/epn2eos_tool/epn2eos/ --output-dir /data/tf/raw --max-tf-per-file 2000 " -else - KR_CONFIG="--writer-type ${WRITER_TYPE} " -fi + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" + o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-${PIPEADD},transport=shmem,rateLogging=1'" \ + --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ --configKeyValues "$ARGS_FILES" \ @@ -67,6 +39,8 @@ o2-dpl-raw-proxy $ARGS_ALL \ --lanes $NLANES \ --configKeyValues "$ARGS_FILES" \ --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ - $KR_CONFIG \ + --writer-type EPN \ + --meta-output-dir $EPN2EOS_METAFILES_DIR \ + --output-dir $CALIB_DIR \ | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host localhost \ - | o2-dpl-run --dds ${WORKFLOWMODE_FILE} | grep -v ERROR + | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/testing/detectors/TPC/workflows.desc b/DATA/testing/detectors/TPC/workflows.desc index 1e763d763..ccb65f2bf 100644 --- a/DATA/testing/detectors/TPC/workflows.desc +++ b/DATA/testing/detectors/TPC/workflows.desc @@ -1,3 +1,4 @@ TPC-krypton: "O2PDPSuite" reco,60,60,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-krypton.sh" +TPC-krypton-raw: "O2PDPSuite" reco,60,60,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-krypton-raw.sh" TPC-laser-raw-filter: "O2PDPSuite" reco,80,80,"SHMSIZE=128000000000 testing/detectors/TPC/tpc-laser-raw-filter.sh" TPC-pattern-generator: "O2PDPSuite" reco,1,1,"SHMSIZE=$((112 << 30)) GPUTYPE=HIP GPUMEMSIZE=$(( 24 << 30 )) HOSTMEMSIZE=$(( 5 << 30 )) testing/detectors/TPC/tpc-pattern-generator.sh" From 091a3295cdc597d0ba79d0498b6adb99f8fa9e29 Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Fri, 27 Oct 2023 15:46:11 +0200 Subject: [PATCH 0158/1239] Add track-extra-converter dependency. (#1304) * Add track-extra-converter dependency. --------- Co-authored-by: Mattia Faggin --- MC/analysis_testing/o2dpg_analysis_test_workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index a3a3e37c8..703c8d446 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -189,6 +189,7 @@ def get_additional_workflows(input_aod): found_O2collision_001 = False found_O2zdc_001 = False found_O2bc_001 = False + found_O2trackextra_001 = False for i in froot.GetListOfKeys(): if "DF_" not in i.GetName(): continue @@ -202,12 +203,16 @@ def get_additional_workflows(input_aod): found_O2zdc_001 = True if "O2bc_001" in j.GetName(): found_O2bc_001 = True + if "O2trackextra_001" in j.GetName(): + found_O2trackextra_001 = True if not found_O2collision_001: additional_workflows.append("o2-analysis-collision-converter --doNotSwap") if not found_O2zdc_001: additional_workflows.append("o2-analysis-zdc-converter") if not found_O2bc_001: additional_workflows.append("o2-analysis-bc-converter") + if not found_O2trackextra_001: + additional_workflows.append("o2-analysis-tracks-extra-converter") break return additional_workflows From b22c613c56f45bf116447ff6a46098143415b235 Mon Sep 17 00:00:00 2001 From: Bong-Hwi Lim Date: Fri, 27 Oct 2023 15:50:01 +0200 Subject: [PATCH 0159/1239] Update Resonance Injector (#1285) * Trigger and Split energy and use json inputfile * Update the test * Remove K892 phi and add delta --- ....ini => GeneratorLF_Resonances_pp1360.ini} | 6 +- .../ini/GeneratorLF_Resonances_pp900.ini | 10 + .../ini/tests/GeneratorLF_Resonances_pp1360.C | 186 ++++++++++++++++++ ...es_pp.C => GeneratorLF_Resonances_pp900.C} | 58 ++++-- .../pythia8/generator/resonancelistgun.json | 182 +++++++++++++++++ .../PWGLF/pythia8/generator/resonances.cfg | 2 +- 6 files changed, 424 insertions(+), 20 deletions(-) rename MC/config/PWGLF/ini/{GeneratorLF_Resonances_pp.ini => GeneratorLF_Resonances_pp1360.ini} (62%) create mode 100644 MC/config/PWGLF/ini/GeneratorLF_Resonances_pp900.ini create mode 100644 MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp1360.C rename MC/config/PWGLF/ini/tests/{GeneratorLF_Resonances_pp.C => GeneratorLF_Resonances_pp900.C} (70%) create mode 100644 MC/config/PWGLF/pythia8/generator/resonancelistgun.json diff --git a/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp1360.ini similarity index 62% rename from MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini rename to MC/config/PWGLF/ini/GeneratorLF_Resonances_pp1360.ini index 073adad06..928bb65b9 100644 --- a/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp.ini +++ b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp1360.ini @@ -1,9 +1,9 @@ [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C -funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonancelist.gun") +funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonancelistgun.json", true, 4) -# [GeneratorPythia8] # before for transport code! -# config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonances.cfg +# [GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg [DecayerPythia8] # after for transport code! config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp900.ini b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp900.ini new file mode 100644 index 000000000..79df3c36f --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLF_Resonances_pp900.ini @@ -0,0 +1,10 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +funcName=generateLF("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonancelistgun.json", true, 4) + +# [GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_pp900gev.cfg + +[DecayerPythia8] # after for transport code! +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg +config[1]=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/resonances.cfg \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp1360.C b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp1360.C new file mode 100644 index 000000000..121db3d0b --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp1360.C @@ -0,0 +1,186 @@ +int External() +{ + std::string path{"o2sim_Kine.root"}; + int numberOfInjectedSignalsPerEvent{1}; + int numberOfGapEvents{4}; + int numberOfEventsProcessed{0}; + int numberOfEventsProcessedWithoutInjection{0}; + std::vector injectedPDGs = { + 323, // K*+- + -323, // K*bar+- + 9010221, // f_0(980) + 113, // rho(770)0 + 213, // rho(770)+ + -213, // rho(770)bar- + 3224, // Sigma(1385)+ + -3224, // Sigma(1385)bar- + 3114, // Sigma(1385)- + -3114, // Sigma(1385)bar+ + 3124, // Lambda(1520)0 + -3124, // Lambda(1520)0bar + 3324, // Xi(1530)0 + -3324, // Xi(1530)0bar + 10323, // K1(1270)+ + -10323, // K1(1270)-bar + 2224, // Delta(1232)+ + -2224, // Delta(1232)bar- + 2114, // Delta(1232)0 + -2114 // Delta(1232)0bar + }; + // TODO: add decay daughters + // 9030221, // f_0(1500) + // 10331, // f_0(1710) + // 123314, // Xi(1820)- + // -123314, // Xi(1820)+ + // 123324, // Xi(1820)0 + // -123324 // Xi(1820)0bar + std::vector> decayDaughters = { + {311, 211}, // K*+- + {-311, -211}, // K*bar+- + {211, -211}, // f_0(980) + {211, -211}, // rho(770)0 + {211, 111}, // rho(770)+ + {-211, 111}, // rho(770)bar- + {3122, 211}, // Sigma(1385)+ + {-3122, -211}, // Sigma(1385)bar- + {3122, -211}, // Sigma(1385)- + {-3122, 211}, // Sigma(1385)bar+ + {2212, -321}, // Lambda(1520)0 + {-2212, 321}, // Lambda(1520)0bar + {3312, 211}, // Xi(1530)0 + {-3312, -211}, // Xi(1530)0bar + {321, 211}, // K1(1270)+ + {-321, -211}, // K1(1270)-bar + {2212, 211}, // Delta(1232)+ + {-2212, -211}, // Delta(1232)bar- + {2212, -211}, // Delta(1232)- + {-2212, 211} // Delta(1232)bar+ + }; + // TODO: add decay daughters + // {211, 211}, // f_0(1500) + // {211, 211}, // f_0(1710) + // {211, 211, 111}, // Xi(1820)- + // {-211, -211, -111}, // Xi(1820)+ + // {211, 211, 111}, // Xi(1820)0 + // {-211, -211, -111} // Xi(1820)0bar + + auto nInjection = injectedPDGs.size(); + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) + { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + if (!tree) + { + std::cerr << "Cannot find tree o2sim in file " << path << "\n"; + return 1; + } + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + std::vector nSignal; + for (int i = 0; i < nInjection; i++) + { + nSignal.push_back(0); + } + std::vector> nDecays; + std::vector nNotDecayed; + for (int i = 0; i < nInjection; i++) + { + std::vector nDecay; + for (int j = 0; j < decayDaughters[i].size(); j++) + { + nDecay.push_back(0); + } + nDecays.push_back(nDecay); + nNotDecayed.push_back(0); + } + auto nEvents = tree->GetEntries(); + bool hasInjection = false; + for (int i = 0; i < nEvents; i++) + { + hasInjection = false; + numberOfEventsProcessed++; + auto check = tree->GetEntry(i); + for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) + { + auto track = tracks->at(idxMCTrack); + auto pdg = track.GetPdgCode(); + auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); + int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG + if (it != injectedPDGs.end()) // found + { + // count signal PDG + nSignal[index]++; + if(track.getFirstDaughterTrackId() < 0) + { + nNotDecayed[index]++; + continue; + } + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) + { + auto pdgDau = tracks->at(j).GetPdgCode(); + bool foundDau= false; + // count decay PDGs + for (int idxDaughter = 0; idxDaughter < decayDaughters[index].size(); ++idxDaughter) + { + if (pdgDau == decayDaughters[index][idxDaughter]) + { + nDecays[index][idxDaughter]++; + foundDau= true; + hasInjection = true; + break; + } + } + if (!foundDau) { + std::cerr << "Decay daughter not found: " << pdg << " -> " << pdgDau << "\n"; + } + } + } + } + if (!hasInjection) + { + numberOfEventsProcessedWithoutInjection++; + } + } + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + for (int i = 0; i < nInjection; i++) + { + std::cout << "# Mother \n"; + std::cout << injectedPDGs[i] << " generated: " << nSignal[i] << ", " << nNotDecayed[i] << " did not decay\n"; + if (nSignal[i] == 0){ + std::cerr << "No generated: " << injectedPDGs[i] << "\n"; + // return 1; // At least one of the injected particles should be generated + } + for (int j = 0; j < decayDaughters[i].size(); j++) + { + std::cout << "# Daughter " << decayDaughters[i][j] << ": " << nDecays[i][j] << "\n"; + } + // if (nSignal[i] != nEvents * numberOfInjectedSignalsPerEvent) + // { + // std::cerr << "Number of generated: " << injectedPDGs[i] << ", lower than expected\n"; + // // return 1; // Don't need to return 1, since the number of generated particles is not the same for each event + // } + } + std::cout << "--------------------------------\n"; + std::cout << "Number of events processed: " << numberOfEventsProcessed << "\n"; + std::cout << "Number of input for the gap events: " << numberOfGapEvents << "\n"; + std::cout << "Number of events processed without injection: " << numberOfEventsProcessedWithoutInjection << "\n"; + // injected event + numberOfGapEvents*gap events + injected event + numberOfGapEvents*gap events + ... + // total fraction of the gap event: numberOfEventsProcessedWithoutInjection/numberOfEventsProcessed + float ratioOfNormalEvents = numberOfEventsProcessedWithoutInjection/numberOfEventsProcessed; + if (ratioOfNormalEvents > 0.75) + { + std::cout << "The number of injected event is loo low!!" << std::endl; + return 1; + } + + return 0; +} + +void GeneratorLF_Resonances_pp1360() { External(); } diff --git a/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp900.C similarity index 70% rename from MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C rename to MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp900.C index d87ec110c..64463dcbe 100644 --- a/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp.C +++ b/MC/config/PWGLF/ini/tests/GeneratorLF_Resonances_pp900.C @@ -1,13 +1,13 @@ int External() { std::string path{"o2sim_Kine.root"}; - int numberOfInjectedSignalsPerEvent{10}; + int numberOfInjectedSignalsPerEvent{1}; + int numberOfGapEvents{4}; + int numberOfEventsProcessed{0}; + int numberOfEventsProcessedWithoutInjection{0}; std::vector injectedPDGs = { - 313, // K0*0 - -313, // K0*0bar 323, // K*+- -323, // K*bar+- - 333, // phi 9010221, // f_0(980) 113, // rho(770)0 213, // rho(770)+ @@ -21,7 +21,11 @@ int External() 3324, // Xi(1530)0 -3324, // Xi(1530)0bar 10323, // K1(1270)+ - -10323 // K1(1270)-bar + -10323, // K1(1270)-bar + 2224, // Delta(1232)+ + -2224, // Delta(1232)bar- + 2114, // Delta(1232)0 + -2114 // Delta(1232)0bar }; // TODO: add decay daughters // 9030221, // f_0(1500) @@ -31,11 +35,8 @@ int External() // 123324, // Xi(1820)0 // -123324 // Xi(1820)0bar std::vector> decayDaughters = { - {321, -211}, // K0*0 - {-321, 211}, // K0*0bar {311, 211}, // K*+- {-311, -211}, // K*bar+- - {321, 321}, // phi {211, -211}, // f_0(980) {211, -211}, // rho(770)0 {211, 111}, // rho(770)+ @@ -49,7 +50,11 @@ int External() {3312, 211}, // Xi(1530)0 {-3312, -211}, // Xi(1530)0bar {321, 211}, // K1(1270)+ - {-321, -211} // K1(1270)-bar + {-321, -211}, // K1(1270)-bar + {2212, 211}, // Delta(1232)+ + {-2212, -211}, // Delta(1232)bar- + {2212, -211}, // Delta(1232)- + {-2212, 211} // Delta(1232)bar+ }; // TODO: add decay daughters // {211, 211}, // f_0(1500) @@ -95,8 +100,11 @@ int External() nNotDecayed.push_back(0); } auto nEvents = tree->GetEntries(); + bool hasInjection = false; for (int i = 0; i < nEvents; i++) { + hasInjection = false; + numberOfEventsProcessed++; auto check = tree->GetEntry(i); for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) { @@ -124,6 +132,7 @@ int External() { nDecays[index][idxDaughter]++; foundDau= true; + hasInjection = true; break; } } @@ -133,6 +142,10 @@ int External() } } } + if (!hasInjection) + { + numberOfEventsProcessedWithoutInjection++; + } } std::cout << "--------------------------------\n"; std::cout << "# Events: " << nEvents << "\n"; @@ -142,19 +155,32 @@ int External() std::cout << injectedPDGs[i] << " generated: " << nSignal[i] << ", " << nNotDecayed[i] << " did not decay\n"; if (nSignal[i] == 0){ std::cerr << "No generated: " << injectedPDGs[i] << "\n"; - return 1; // At least one of the injected particles should be generated + // return 1; // At least one of the injected particles should be generated } for (int j = 0; j < decayDaughters[i].size(); j++) { std::cout << "# Daughter " << decayDaughters[i][j] << ": " << nDecays[i][j] << "\n"; } - if (nSignal[i] != nEvents * numberOfInjectedSignalsPerEvent) - { - std::cerr << "Number of generated: " << injectedPDGs[i] << ", lower than expected\n"; - // return 1; // Don't need to return 1, since the number of generated particles is not the same for each event - } + // if (nSignal[i] != nEvents * numberOfInjectedSignalsPerEvent) + // { + // std::cerr << "Number of generated: " << injectedPDGs[i] << ", lower than expected\n"; + // // return 1; // Don't need to return 1, since the number of generated particles is not the same for each event + // } + } + std::cout << "--------------------------------\n"; + std::cout << "Number of events processed: " << numberOfEventsProcessed << "\n"; + std::cout << "Number of input for the gap events: " << numberOfGapEvents << "\n"; + std::cout << "Number of events processed without injection: " << numberOfEventsProcessedWithoutInjection << "\n"; + // injected event + numberOfGapEvents*gap events + injected event + numberOfGapEvents*gap events + ... + // total fraction of the gap event: numberOfEventsProcessedWithoutInjection/numberOfEventsProcessed + float ratioOfNormalEvents = numberOfEventsProcessedWithoutInjection/numberOfEventsProcessed; + if (ratioOfNormalEvents > 0.75) + { + std::cout << "The number of injected event is loo low!!" << std::endl; + return 1; } + return 0; } -void GeneratorLF_Resonances_pp() { External(); } +void GeneratorLF_Resonances_pp900() { External(); } diff --git a/MC/config/PWGLF/pythia8/generator/resonancelistgun.json b/MC/config/PWGLF/pythia8/generator/resonancelistgun.json new file mode 100644 index 000000000..5ec4ec892 --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/resonancelistgun.json @@ -0,0 +1,182 @@ +{ + "K(892)+" : { + "pdg": 323, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "K(892)-" : { + "pdg": -323, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "f_0(980)" : { + "pdg": 9010221, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "rho(770)0" : { + "pdg": 113, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "rho(770)+" : { + "pdg": 213, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "rho(770)-" : { + "pdg": -213, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Sigma(1385)-" : { + "pdg": 3114, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Sigma(1385)+" : { + "pdg": -3114, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Sigma(1385)+" : { + "pdg": 3224, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Sigma(1385)-" : { + "pdg": -3224, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Lambda(1520)0" : { + "pdg": 3124, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Lambda(1520)0" : { + "pdg": -3124, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Xi(1530)0" : { + "pdg": 3324, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Xi(1530)0" : { + "pdg": -3324, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "K1(1270)+": { + "pdg": 10323, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "K1(1270)-": { + "pdg": -10323, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Delta++" : { + "pdg": 2224, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Delta++" : { + "pdg": -2224, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "Delta0" : { + "pdg": 2114, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + }, + "anti-Delta0" : { + "pdg": -2114, + "n": 1, + "ptMin": 0.0, + "ptMax": 20, + "etaMin": -1.2, + "etaMax": 1.2, + "genDecayed": true + } +} \ No newline at end of file diff --git a/MC/config/PWGLF/pythia8/generator/resonances.cfg b/MC/config/PWGLF/pythia8/generator/resonances.cfg index df226c5fc..6069b5a90 100644 --- a/MC/config/PWGLF/pythia8/generator/resonances.cfg +++ b/MC/config/PWGLF/pythia8/generator/resonances.cfg @@ -2,7 +2,7 @@ ProcessLevel:all = off # will not look for the 'process' # id::all = name antiName spinType chargeType colType m0 mWidth mMin mMax tau0 -3124:all = Lambda1520 Lambda1520bar 0 0 1 1.5195 0.01560 0.06240 0.00000E+00 0 1 +3124:all = Lambda1520 Lambda1520bar 0 0 0 1.5195 0.01560 0.06240 0.00000E+00 0 1 ### add Resonance decays absent in PYTHIA8 decay table and set BRs from PDG for other 3124:oneChannel = 1 0.223547 0 2212 -321 From f73610eca015d3a80de32ba2438268a6e729b032 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 27 Oct 2023 13:28:54 +0200 Subject: [PATCH 0160/1239] Fix determination of seed --- MC/bin/o2dpg_sim_workflow_anchored.py | 4 ++-- MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index ad9ba4a0a..027d6631e 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -282,7 +282,7 @@ def main(): args = parser.parse_args() # split id should not be larger than production id - assert(args.split_id < args.prod_split) + assert(args.split_id <= args.prod_split) # make a CCDB accessor object ccdbreader = CCDBAccessor(args.ccdb_url) @@ -295,7 +295,7 @@ def main(): # determine timestamp, and production offset for the final # MC job to run - timestamp, prod_offset = determine_timestamp(GLOparams["SOR"], GLOparams["EOR"], [args.split_id, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) + timestamp, prod_offset = determine_timestamp(GLOparams["SOR"], GLOparams["EOR"], [args.split_id - 1, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) # this is anchored to print ("Determined start-of-run to be: ", GLOparams["SOR"]) print ("Determined timestamp to be : ", timestamp) diff --git a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh index 814b832c2..3289e6e15 100755 --- a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh +++ b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh @@ -89,7 +89,7 @@ NSIGEVENTS=${NSIGEVENTS:-22} # THIS NEEDS TO COME FROM OUTSIDE # echo "$" | awk -F' -- ' '{print $1, $3}' -baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" +baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ From 30b4181299069389e131a082e8c781b2deb70cd2 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 30 Oct 2023 11:51:29 +0100 Subject: [PATCH 0161/1239] Adjust split ID --- MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh index 7a5ca1e4e..01563e70b 100755 --- a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh +++ b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh @@ -99,7 +99,7 @@ NSIGEVENTS=${NSIGEVENTS:-22} # THIS NEEDS TO COME FROM OUTSIDE # echo "$" | awk -F' -- ' '{print $1, $3}' -baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-0} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" +baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${ALIEN_JDL_PRODSPLIT:-100} --run-number ${RUNNUMBER} -eCM 900 -col pp" # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ From 8886c931499bae96d7ee183e1a4ee55061afffd0 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 30 Oct 2023 16:19:55 +0100 Subject: [PATCH 0162/1239] 5 threads for sec vtx in case of PbPb with 2 TF in flight --- DATA/production/configurations/asyncReco/async_pass.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 8e879a06d..cb6e4608c 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -409,6 +409,7 @@ else export TIMEFRAME_RATE_LIMIT=2 export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu export SHMSIZE=16000000000 + export SVERTEX_THREADS=5 fi else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs From 04be4fd254120b59e4fcf72d9c9ec22eece0ad5e Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Mon, 30 Oct 2023 17:57:07 +0100 Subject: [PATCH 0163/1239] [EMCAL-756] Add missing subspec in BC task in config for PbPb --- DATA/production/qc-async/emc_PbPb.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index 8706a1f89..17a0c5577 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -87,7 +87,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "emcal-triggers:EMC/CELLSTRGR;ctp-digits:CTP/DIGITS" + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS" }, "taskParameters": { "AliasMB" : "CMTVXTSC" From b8a2612c1de127b7ca7a531633d6fc2f8ac02e02 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 30 Oct 2023 09:30:08 +0100 Subject: [PATCH 0164/1239] Updates to be able to run async reco workflow in split mode --- .../configurations/asyncReco/async_pass.sh | 26 +++++++++++-------- .../configurations/asyncReco/setenv_extra.sh | 6 +++++ DATA/production/qc-async/mch-tracks.json | 4 +-- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index cb6e4608c..18676ccf1 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -462,18 +462,18 @@ else echo "We will run the workflow in SPLIT mode!" WORKFLOW_PARAMETERS_START=$WORKFLOW_PARAMETERS - if [[ -z "$ALIEN_JDL_SPLITSTEP" ]] || [[ "$ALIEN_JDL_SPLITSTEP" -eq 1 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 1 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq "all" ]]; then + if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 1 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 1 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" == "all" ]]; then # 1. TPC decoding + reco - echo "Step 1) Decoding and reconstructing TPC" - echo "Step 1) Decoding and reconstructing TPC" > workflowconfig.log + echo "Step 1) Decoding and reconstructing TPC+CTP" + echo "Step 1) Decoding and reconstructing TPC+CTP" > workflowconfig.log for i in AOD QC CALIB CALIB_LOCAL_INTEGRATED_AGGREGATOR; do export WORKFLOW_PARAMETERS=$(echo $WORKFLOW_PARAMETERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") done - env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=TPC WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=TPC,CTP WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then timeStart=`date +%s` - time env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=TPC WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + time env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=TPC,CTP WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list exitcode=$? timeEnd=`date +%s` timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) @@ -490,7 +490,7 @@ else fi fi - if [[ -z "$ALIEN_JDL_SPLITSTEP" ]] || [[ "$ALIEN_JDL_SPLITSTEP" -eq 2 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 2 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq "all" ]]; then + if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 2 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 2 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" == "all" ]]; then # 2. the other detectors decoding + reco WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START echo "Step 2) Decoding and reconstructing ALL-TPC" @@ -498,11 +498,11 @@ else for i in AOD QC CALIB CALIB_LOCAL_INTEGRATED_AGGREGATOR; do export WORKFLOW_PARAMETERS=$(echo $WORKFLOW_PARAMETERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") done - env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=TPC WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=TPC,$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then timeStart=`date +%s` - time env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=TPC WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + time env DISABLE_ROOT_OUTPUT=0 IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=TPC,$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_MATCHING= ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list exitcode=$? timeEnd=`date +%s` timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) @@ -542,17 +542,21 @@ else fi fi - if [[ -z "$ALIEN_JDL_SPLITSTEP" ]] || [[ "$ALIEN_JDL_SPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq "all" ]]; then + if ([[ -z "$ALIEN_JDL_SPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq "all" ]]; then # 3. matching, QC, calib, AOD WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START echo "Step 3) matching, QC, calib, AOD" echo -e "\nStep 3) matching, QC, calib, AOD" >> workflowconfig.log export TIMEFRAME_RATE_LIMIT=0 - env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE_QC=CPV ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + echo "Removing detectors $DETECTORS_EXCLUDE" + READER_DELAY=${ALIEN_JDL_READERDELAY:-30} + export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " + echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader_workflow" + env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then timeStart=`date +%s` - time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE_QC=CPV ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list exitcode=$? timeEnd=`date +%s` timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index ccc019af4..8d8636f32 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -21,6 +21,12 @@ else # removing MID for these runs: it was noisy and therefore declared bad, and makes the reco crash export WORKFLOW_DETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MFT,MCH,TRD,EMC,PHS,CPV,HMP,ZDC,CTP fi + # list of detectors to possibly exclude + if [[ -n $ALIEN_JDL_DETECTORSEXCLUDE ]]; then + echo "ALIEN_JDL_DETECTORSEXCLUDE = $ALIEN_JDL_DETECTORSEXCLUDE" + export DETECTORS_EXCLUDE=$ALIEN_JDL_DETECTORSEXCLUDE # will be used in the async_pass.sh if we run in split mode + export WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE + fi fi # ad-hoc settings for CTF reader: we are on the grid, we read the files remotely diff --git a/DATA/production/qc-async/mch-tracks.json b/DATA/production/qc-async/mch-tracks.json index e5c5d729f..137486fa7 100644 --- a/DATA/production/qc-async/mch-tracks.json +++ b/DATA/production/qc-async/mch-tracks.json @@ -33,7 +33,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS" + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS" }, "taskParameters": { "maxTracksPerTF": "600", @@ -52,4 +52,4 @@ } } } -} \ No newline at end of file +} From 3332e24452d686dbe31586275ced2f7fb95ffeab Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 30 Oct 2023 12:10:39 +0100 Subject: [PATCH 0165/1239] Removing option that does not exists anymore --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 8d8636f32..1513bff4a 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -493,7 +493,7 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 export CONFIG_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="scdcalib.maxTracksPerCalibSlot=35000000;scdcalib.minPtNoOuterPoint=0.2;scdcalib.maxQ2Pt=5;scdcalib.minITSNClsNoOuterPoint=6;scdcalib.minITSNCls=4;scdcalib.minTPCNClsNoOuterPoint=90" - export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds --enable-itsonly" + export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds" # ad-hoc settings for TPC residual extraction export ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator="$ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator --output-type trackParams,unbinnedResid" if [[ $ALIEN_JDL_DEBUGRESIDUALEXTRACTION == "1" ]]; then From 29cc64b53641516f0d3094295c9c24266da4396d Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Mon, 30 Oct 2023 17:46:47 +0100 Subject: [PATCH 0166/1239] Enable CTP readout replay at AOD level for triggered detectors --- MC/bin/o2dpg_sim_workflow.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index b6c7e0152..c2b980cc8 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1378,6 +1378,10 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if not args.with_strangeness_tracking: AODtask['cmd'] += ' --disable-strangeness-tracker' + # Enable CTP readout replay for triggered detectors (EMCAL, HMPID, PHOS/CPV, TRD) + # Needed untill triggers are supported in CTP simulation + AODtask['cmd'] += ' --ctpreadout-create 1' + workflow['stages'].append(AODtask) # AOD merging / combination step (as individual stages) --> for the moment deactivated in favor or more stable global merging From da2ec57a46a556d497ed567ca87e1a4717c6d877 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Wed, 1 Nov 2023 12:53:54 +0100 Subject: [PATCH 0167/1239] [EMCAL-1054] Include EMCAL cell recalibrator workflow for bad channel masking (#1312) - Chain cell-recalibrator workfow with reconstruction workflow to reject bad channels seen in data - Needs to disable cell writer within reconstruction workflow and extra cell writer at the end of the full chain --- MC/bin/o2dpg_sim_workflow.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index c2b980cc8..9db08a754 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1050,7 +1050,11 @@ def getDigiTaskName(det): # calorimeters EMCRECOtask = createTask(name='emcalreco_'+str(tf), needs=[getDigiTaskName("EMC")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500') - EMCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-emcal-reco-workflow --input-type digits --output-type cells --infile emcaldigits.root ' + getDPL_global_options(ccdbbackend=False) + putConfigValues() + EMCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-emcal-reco-workflow --input-type digits --output-type cells --infile emcaldigits.root --disable-root-output --subspecificationOut 1 ' + putConfigValues() + EMCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] + EMCRECOtask['cmd'] += ' | ${O2_ROOT}/bin/o2-emcal-cell-recalibrator-workflow --input-subspec 1 --output-subspec 0 --no-timecalib --no-gaincalib ' + putConfigValues() + EMCRECOtask['cmd'] += (' --isMC','')[args.no_mc_labels] + EMCRECOtask['cmd'] += ' | ${O2_ROOT}/bin/o2-emcal-cell-writer-workflow --subspec 0 ' + getDPL_global_options() + putConfigValues() EMCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(EMCRECOtask) From f3df2a2a3abd20996cd0ec5fe3a4b745f2532c97 Mon Sep 17 00:00:00 2001 From: aferrero2707 Date: Fri, 3 Nov 2023 19:05:24 +0100 Subject: [PATCH 0168/1239] MUON: improve SYNC QC for global forward matched tracks (#1314) * [MUON] improve SYNC QC for global forward matched tracks Currently the SYNC QC for matched forward tracks is included in the MCH configuration, and limited to MCH-MID tracks. The commit introduces separate QC configuration files for forward matched tracks in SYNC processing, for three different combinations: - MCH-MID matching (MFT not included) - MFT-MCH matching (MID not included) - full MFT-MCH-MID matching: in this case three set of plots are produced, corresponding to the possible matching combinations (MFT-MCH, MCH-MID and MFT-MCH-MID) * [MUON] add global forward matched tracks QC for local and CI processing --- .../qc-sync/glo-mchmid-mtch-qcmn-epn.json | 66 +++++++++++++++++++ .../qc-sync/glo-mftmch-mtch-qcmn-epn.json | 66 +++++++++++++++++++ .../qc-sync/glo-mftmchmid-mtch-qcmn-epn.json | 66 +++++++++++++++++++ DATA/production/qc-workflow.sh | 14 ++++ 4 files changed, 212 insertions(+) create mode 100644 DATA/production/qc-sync/glo-mchmid-mtch-qcmn-epn.json create mode 100644 DATA/production/qc-sync/glo-mftmch-mtch-qcmn-epn.json create mode 100644 DATA/production/qc-sync/glo-mftmchmid-mtch-qcmn-epn.json diff --git a/DATA/production/qc-sync/glo-mchmid-mtch-qcmn-epn.json b/DATA/production/qc-sync/glo-mchmid-mtch-qcmn-epn.json new file mode 100644 index 000000000..74d113268 --- /dev/null +++ b/DATA/production/qc-sync/glo-mchmid-mtch-qcmn-epn.json @@ -0,0 +1,66 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ali-qcdb.cern.ch:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": {}, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "http://localhost:8500" + }, + "conditionDB": { + "url": "o2-ccdb.internal" + }, + "bookkeeping": { + "url": "alio2-cr1-hv-web01.cern.ch:4001" + } + }, + "tasks": { + "MUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "cycleDurationSeconds": "180", + "maxNumberCycles": "-1", + "disableLastCycle": "true", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;matchMCHMID:GLO/MTC_MCHMID" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID": "MCH,MID,MCH-MID" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "local", + "mergingMode": "delta", + "localControl": "odc", + "localMachines": [ + "localhost", + "epn" + ], + "remotePort": "29514", + "remoteMachine": "alio2-cr1-qts02.cern.ch" + } + }, + "checks": {} + }, + "dataSamplingPolicies": [] +} diff --git a/DATA/production/qc-sync/glo-mftmch-mtch-qcmn-epn.json b/DATA/production/qc-sync/glo-mftmch-mtch-qcmn-epn.json new file mode 100644 index 000000000..0477408e6 --- /dev/null +++ b/DATA/production/qc-sync/glo-mftmch-mtch-qcmn-epn.json @@ -0,0 +1,66 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ali-qcdb.cern.ch:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": {}, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "http://localhost:8500" + }, + "conditionDB": { + "url": "o2-ccdb.internal" + }, + "bookkeeping": { + "url": "alio2-cr1-hv-web01.cern.ch:4001" + } + }, + "tasks": { + "MUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "cycleDurationSeconds": "180", + "maxNumberCycles": "-1", + "disableLastCycle": "true", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMFT:MFT/TRACKS;trackMFTROF:MFT/MFTTrackROF;trackMFTClIdx:MFT/TRACKCLSID;alpparMFT:MFT/ALPIDEPARAM;fwdtracks:GLO/GLFWD" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID": "MFT,MCH,MFT-MCH" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "local", + "mergingMode": "delta", + "localControl": "odc", + "localMachines": [ + "localhost", + "epn" + ], + "remotePort": "29514", + "remoteMachine": "alio2-cr1-qts02.cern.ch" + } + }, + "checks": {} + }, + "dataSamplingPolicies": [] +} diff --git a/DATA/production/qc-sync/glo-mftmchmid-mtch-qcmn-epn.json b/DATA/production/qc-sync/glo-mftmchmid-mtch-qcmn-epn.json new file mode 100644 index 000000000..9ae3afac3 --- /dev/null +++ b/DATA/production/qc-sync/glo-mftmchmid-mtch-qcmn-epn.json @@ -0,0 +1,66 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ali-qcdb.cern.ch:8083", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": {}, + "monitoring": { + "url": "influxdb-unix:///tmp/telegraf.sock" + }, + "consul": { + "url": "http://localhost:8500" + }, + "conditionDB": { + "url": "o2-ccdb.internal" + }, + "bookkeeping": { + "url": "alio2-cr1-hv-web01.cern.ch:4001" + } + }, + "tasks": { + "MUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "cycleDurationSeconds": "180", + "maxNumberCycles": "-1", + "disableLastCycle": "true", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMFT:MFT/TRACKS;trackMFTROF:MFT/MFTTrackROF;trackMFTClIdx:MFT/TRACKCLSID;alpparMFT:MFT/ALPIDEPARAM;fwdtracks:GLO/GLFWD;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;matchMCHMID:GLO/MTC_MCHMID" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID": "MFT,MCH,MID,MFT-MCH,MCH-MID,MFT-MCH-MID" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "local", + "mergingMode": "delta", + "localControl": "odc", + "localMachines": [ + "localhost", + "epn" + ], + "remotePort": "29514", + "remoteMachine": "alio2-cr1-qts02.cern.ch" + } + }, + "checks": {} + }, + "dataSamplingPolicies": [] +} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 90501a593..4d17da67c 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -110,6 +110,13 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then QC_JSON_TOF_MATCH=consul://o2/components/qc/ANY/any/tof-qcmn-match-itstpctof fi fi + if has_detectors_reco MFT MCH MID && has_matching_qc MFTMCH && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=consul://o2/components/qc/ANY/any/glo-mftmchmid-mtch-qcmn-epn + elif has_detectors_reco MFT MCH && has_matching_qc MFTMCH; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=consul://o2/components/qc/ANY/any/glo-mftmch-mtch-qcmn-epn + elif has_detectors_reco MCH MID && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MCHMID:-}" ]] && QC_JSON_GLO_MCHMID=consul://o2/components/qc/ANY/any/glo-mchmid-mtch-qcmn-epn + fi if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then [[ -z "${QC_JSON_GLOBAL:-}" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global-epn-staging.json # this must be last else @@ -146,6 +153,13 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then QC_JSON_TOF_MATCH=$O2DPG_ROOT/DATA/production/qc-sync/itstpctof.json fi fi + if has_detectors_reco MFT MCH MID && has_matching_qc MFTMCH && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=$O2DPG_ROOT/DATA/production/qc-sync/glo-mftmchmid-mtch-qcmn-epn.json + elif has_detectors_reco MFT MCH && has_matching_qc MFTMCH; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=$O2DPG_ROOT/DATA/production/qc-sync/glo-mftmch-mtch-qcmn-epn.json + elif has_detectors_reco MCH MID && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MCHMID:-}" ]] && QC_JSON_GLO_MCHMID=$O2DPG_ROOT/DATA/production/qc-sync/glo-mchmid-mtch-qcmn-epn.json + fi [[ -z "${QC_JSON_GLOBAL:-}" ]] && QC_JSON_GLOBAL=$O2DPG_ROOT/DATA/production/qc-sync/qc-global.json # this must be last QC_CONFIG_OVERRIDE+="qc.config.conditionDB.url=${DPL_CONDITION_BACKEND:-http://alice-ccdb.cern.ch};" From da4299c579675a54f6760a39a13ebd368b02defd Mon Sep 17 00:00:00 2001 From: shahoian Date: Mon, 6 Nov 2023 00:37:06 +0100 Subject: [PATCH 0169/1239] Fixes for PVertexing in PbPb needs https://github.com/AliceO2Group/AliceO2/pull/12207 https://github.com/AliceO2Group/AliceO2/pull/12151 https://github.com/AliceO2Group/AliceO2/pull/12043 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 1513bff4a..c3bf1cdfd 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -407,8 +407,8 @@ if [[ $ALIGNLEVEL == 1 ]]; then if [[ $BEAMTYPE == "pp" ]]; then export PVERTEXER+=";pvertexer.maxChi2TZDebris=40;pvertexer.maxChi2Mean=12;pvertexer.maxMultRatDebris=1.;pvertexer.addTimeSigma2Debris=1e-2;pvertexer.meanVertexExtraErrSelection=0.03;" elif [[ $BEAMTYPE == "PbPb" ]]; then - # at the moment placeholder - export PVERTEXER+=";pvertexer.maxChi2Mean=12;pvertexer.addTimeSigma2Debris=1e-2;pvertexer.meanVertexExtraErrSelection=0.03;" + export PVERTEXER+=";pvertexer.addTimeSigma2Debris=1e-2;pvertexer.meanVertexExtraErrSelection=0.03;pvertexer.maxITSOnlyFraction=0.85;pvertexer.maxTDiffDebris=1.5;pvertexer.maxZDiffDebris=0.3;pvertexer.addZSigma2Debris=0.09;pvertexer.addTimeSigma2Debris=2.25;pvertexer.maxChi2TZDebris=100;pvertexer.maxMultRatDebris=1.;pvertexer.maxTDiffDebrisExtra=-1.;pvertexer.dbscanDeltaT=-0.55;pvertexer.maxTMAD=1.;pvertexer.maxZMAD=0.04;" + has_detector_reco FT0 && PVERTEX_CONFIG+=" --validate-with-ft0 " fi fi From cb840bd7eef5c6133e76c4250d9a5582c9148f03 Mon Sep 17 00:00:00 2001 From: Diego Stocco Date: Mon, 6 Nov 2023 10:15:03 +0100 Subject: [PATCH 0170/1239] Match recent changes in consul (#1235) --- DATA/production/qc-async/mid.json | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/DATA/production/qc-async/mid.json b/DATA/production/qc-async/mid.json index bbad331ab..9c6d6997e 100644 --- a/DATA/production/qc-async/mid.json +++ b/DATA/production/qc-async/mid.json @@ -34,7 +34,7 @@ "dataSource": { "type": "direct", "query": "digits:MID/DATA;digits_rof:MID/DATAROF", - "query_comment" : "100% sampling" + "query_comment": "100% sampling" } }, "MIDClusters": { @@ -48,7 +48,7 @@ "dataSource": { "type": "direct", "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS", - "query_comment" : "100% sampling" + "query_comment": "100% sampling" } }, "MIDTracks": { @@ -62,36 +62,36 @@ "dataSource": { "type": "direct", "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS", - "query_comment" : "100% sampling" + "query_comment": "100% sampling" } } }, "checks": { "MIDDigits": { - "active": "false", + "active": "true", "checkName": "Digits", "className": "o2::quality_control_modules::mid::DigitsQcCheck", "moduleName": "QcMID", "detectorName": "MID", "policy": "OnAny", "checkParameters": { - "MeanMultThreshold": "100." + "MeanMultThreshold": "100.", + "MinMultThreshold": "0.0", + "NbOrbitPerTF": "32.", + "LocalBoardScale": "200.0", + "LocalBoardThreshold": "800.0", + "NbBadLocalBoard": "10.", + "NbEmptyLocalBoard": "117." }, "dataSource": [ { "type": "Task", - "name": "MIDDigits", - "MOs": [ - "mMultHitMT11B", - "mMultHitMT12B", - "mMultHitMT21B", - "mMultHitMT22B" - ] + "name": "MIDDigits" } ] }, "MIDClusters": { - "active": "false", + "active": "true", "checkName": "Clusters", "className": "o2::quality_control_modules::mid::ClustQcCheck", "moduleName": "QcMID", @@ -100,28 +100,28 @@ "dataSource": [ { "type": "Task", - "name": "MIDClusters", - "MOs": [] + "name": "MIDClusters" } ] }, "MIDTracks": { - "active": "false", + "active": "true", "checkName": "Tracks", "className": "o2::quality_control_modules::mid::TracksQcCheck", "moduleName": "QcMID", "detectorName": "MID", "policy": "OnAny", + "checkParameters": { + "Ratio44Threshold": "0.1" + }, "dataSource": [ { "type": "Task", - "name": "MIDTracks", - "MOs": [] + "name": "MIDTracks" } ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file From 1c33c679c4658e203dc1611bdd8fa8b53cda970b Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Mon, 6 Nov 2023 11:01:03 +0100 Subject: [PATCH 0171/1239] Change default setting of mult. axis for trackQC. (#1315) * Change default setting of mult. axis for trackQC. --> Remove setting for binsTrackMultiplicity to use default. --------- Co-authored-by: Mattia Faggin --- .../json/EventSelectionQA/pbpb/analysis-testing-data.json | 7 ------- .../json/EventTrackQA/pbpb/analysis-testing-data.json | 7 ------- .../json/EventTrackQA/pp/analysis-testing-data.json | 7 ------- .../json/default/pbpb/analysis-testing-data.json | 7 ------- .../json/default/pbpb/analysis-testing-mc.json | 7 ------- .../json/default/pp/analysis-testing-data.json | 7 ------- .../json/default/pp/analysis-testing-mc.json | 7 ------- 7 files changed, 49 deletions(-) diff --git a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json index 4823f8664..c138798fe 100644 --- a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json @@ -296,13 +296,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json index 52c800a8b..17ca41a62 100644 --- a/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json @@ -37,13 +37,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json index 52c800a8b..17ca41a62 100644 --- a/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json @@ -37,13 +37,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json index 6b1ad0ad5..ba6042a73 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json @@ -295,13 +295,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json index 62209a027..844fdebb6 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json @@ -1055,13 +1055,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json index 6b1ad0ad5..ba6042a73 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -295,13 +295,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json index 62209a027..844fdebb6 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -1055,13 +1055,6 @@ "50" ] }, - "binsTrackMultiplcity": { - "values": [ - "200", - "0", - "200" - ] - }, "binsVertexPosXY": { "values": [ "500", From 276cf8042a3808f1eae1eb780e49bce1c2b57c35 Mon Sep 17 00:00:00 2001 From: aferrero2707 Date: Fri, 3 Nov 2023 13:41:59 +0100 Subject: [PATCH 0172/1239] [MUON] add ASYNC QC for global forward matched tracks The commit introduces QC plots for forward matched tracks in ASYC processing, for three different configurations: - MCH-MID matching (MFT not included) - MFT-MCH matching (MID not included) - full MFT-MCH-MID matching: in this case three set of plots are produced, corresponding to the possible matching combinations (MFT-MCH, MCH-MID and MFT-MCH-MID) --- DATA/production/qc-async/mchmid-tracks.json | 34 +++++++++++++++++++ DATA/production/qc-async/mftmch-tracks.json | 34 +++++++++++++++++++ .../production/qc-async/mftmchmid-tracks.json | 34 +++++++++++++++++++ DATA/production/qc-workflow.sh | 7 ++++ 4 files changed, 109 insertions(+) create mode 100644 DATA/production/qc-async/mchmid-tracks.json create mode 100644 DATA/production/qc-async/mftmch-tracks.json create mode 100644 DATA/production/qc-async/mftmchmid-tracks.json diff --git a/DATA/production/qc-async/mchmid-tracks.json b/DATA/production/qc-async/mchmid-tracks.json new file mode 100644 index 000000000..2de4b479a --- /dev/null +++ b/DATA/production/qc-async/mchmid-tracks.json @@ -0,0 +1,34 @@ +{ + "qc": { + "tasks": { + "TaskMUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "taskName": "MUONTracks", + "cycleDurationSeconds": "300", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;matchMCHMID:GLO/MTC_MCHMID" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID" : "MCH,MID,MCH-MID" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "true", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "remote" + } + } + } +} diff --git a/DATA/production/qc-async/mftmch-tracks.json b/DATA/production/qc-async/mftmch-tracks.json new file mode 100644 index 000000000..02b6a9def --- /dev/null +++ b/DATA/production/qc-async/mftmch-tracks.json @@ -0,0 +1,34 @@ +{ + "qc": { + "tasks": { + "TaskMUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "taskName": "MUONTracks", + "cycleDurationSeconds": "300", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMFT:MFT/TRACKS;trackMFTROF:MFT/MFTTrackROF;trackMFTClIdx:MFT/TRACKCLSID;alpparMFT:MFT/ALPIDEPARAM;fwdtracks:GLO/GLFWD" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID" : "MCH,MFT,MFT-MCH" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "true", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "remote" + } + } + } +} diff --git a/DATA/production/qc-async/mftmchmid-tracks.json b/DATA/production/qc-async/mftmchmid-tracks.json new file mode 100644 index 000000000..90eebdb92 --- /dev/null +++ b/DATA/production/qc-async/mftmchmid-tracks.json @@ -0,0 +1,34 @@ +{ + "qc": { + "tasks": { + "TaskMUONTracks": { + "active": "true", + "className": "o2::quality_control_modules::muon::TracksTask", + "moduleName": "QcMUONCommon", + "detectorName": "GLO", + "taskName": "MUONTracks", + "cycleDurationSeconds": "300", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "trackMCH:MCH/TRACKS;trackMCHROF:MCH/TRACKROFS;trackMCHTRACKCLUSTERS:MCH/TRACKCLUSTERS;mchtrackdigits:MCH/CLUSTERDIGITS;trackMFT:MFT/TRACKS;trackMFTROF:MFT/MFTTrackROF;trackMFTClIdx:MFT/TRACKCLSID;alpparMFT:MFT/ALPIDEPARAM;fwdtracks:GLO/GLFWD;trackMID:MID/TRACKS;trackMIDROF:MID/TRACKROFS;trackMIDTRACKCLUSTERS:MID/TRACKCLUSTERS;trackClMIDROF:MID/TRCLUSROFS;matchMCHMID:GLO/MTC_MCHMID" + }, + "taskParameters": { + "maxTracksPerTF": "600", + "GID" : "MFT,MCH,MID,MFT-MCH,MCH-MID,MFT-MCH-MID" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "true", + "askGRPLHCIF": "false", + "askGRPMagField": "false", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "false", + "needPropagatorD": "false" + }, + "location": "remote" + } + } + } +} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 4d17da67c..0a3ce917d 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -190,6 +190,13 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then add_QC_JSON MCH_TRACKS $O2DPG_ROOT/DATA/production/qc-async/mch-tracks.json fi fi + if has_detectors_reco MFT MCH MID && has_matching_qc MFTMCH && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=$O2DPG_ROOT/DATA/production/qc-async/mftmchmid-tracks.json + elif has_detectors_reco MFT MCH && has_matching_qc MFTMCH; then + [[ -z "${QC_JSON_GLO_MFTMCH:-}" ]] && QC_JSON_GLO_MFTMCH=$O2DPG_ROOT/DATA/production/qc-async/mftmch-tracks.json + elif has_detectors_reco MCH MID && has_matching_qc MCHMID; then + [[ -z "${QC_JSON_GLO_MCHMID:-}" ]] && QC_JSON_GLO_MCHMID=$O2DPG_ROOT/DATA/production/qc-async/mchmid-tracks.json + fi [[ -z "${QC_JSON_CPV:-}" ]] && QC_JSON_CPV=$O2DPG_ROOT/DATA/production/qc-async/cpv.json [[ -z "${QC_JSON_PHS:-}" ]] && QC_JSON_PHS=$O2DPG_ROOT/DATA/production/qc-async/phs.json [[ -z "${QC_JSON_TRD:-}" ]] && QC_JSON_TRD=$O2DPG_ROOT/DATA/production/qc-async/trd.json From 0704de4aa665015ee2f51a206ec41ac8b4eb049d Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 6 Nov 2023 16:26:55 +0100 Subject: [PATCH 0173/1239] Fixing condition to setup env for EPNs --- DATA/production/common/getStat.sh | 1 + .../configurations/asyncReco/async_pass.sh | 53 +++++++++++++++---- .../configurations/asyncReco/setenv_extra.sh | 2 +- DATA/production/workflow-multiplicities.sh | 24 ++++----- 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/DATA/production/common/getStat.sh b/DATA/production/common/getStat.sh index 5ae1bf9de..29a0fdbaa 100755 --- a/DATA/production/common/getStat.sh +++ b/DATA/production/common/getStat.sh @@ -14,6 +14,7 @@ nCTFsProcessed=$((nCTFsProcessed + 1)) if [[ $nCTFsFilesInspected != $((nCTFsFilesFailed + nCTFsFilesOK)) ]]; then echo "Something went wrong with parsing the log file: CTF files inspected ($nCTFsFilesInspected) is not the sum of those successfully processed ($nCTFsFilesOK) and those that failed ($nCTFsFilesFailed)" + exit 8 fi while read -r line; do currentPVs=`echo $line | sed 's/^.*Found \([0-9]*\) PVs.*/\1/'` diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 18676ccf1..2999b3e43 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -353,7 +353,7 @@ fi echo "SETTING_ROOT_OUTPUT = $SETTING_ROOT_OUTPUT" # Enabling GPUs -if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]]; then +if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then echo "Enabling GPUS" export GPUTYPE="HIP" export GPUMEMSIZE=$((25 << 30)) @@ -456,6 +456,13 @@ if [[ $ALIEN_JDL_SPLITWF != "1" ]]; then fi mv latest.log latest_reco_1.log $STATSCRIPT latest_reco_1.log + exitcode=$? + echo "exit code is $exitcode" + if [[ $exitcode -ne 0 ]]; then + echo "exit code from processing is " $exitcode > validation_error.message + echo "exit code from processing is " $exitcode + exit $exitcode + fi fi else # running the wf in split mode @@ -486,7 +493,16 @@ else exit $exitcode fi mv latest.log latest_reco_1.log + if [[ -f performanceMetrics.json ]]; then + mv performanceMetrics.json performanceMetrics_1.json + fi $STATSCRIPT latest_reco_1.log reco_1 + exitcode=$? + if [[ $exitcode -ne 0 ]]; then + echo "exit code from processing is " $exitcode > validation_error.message + echo "exit code from processing is " $exitcode + exit $exitcode + fi fi fi @@ -515,7 +531,16 @@ else exit $exitcode fi mv latest.log latest_reco_2.log + if [[ -f performanceMetrics.json ]]; then + mv performanceMetrics.json performanceMetrics_2.json + fi $STATSCRIPT latest_reco_2.log reco_2 + exitcode=$? + if [[ $exitcode -ne 0 ]]; then + echo "exit code from processing is " $exitcode > validation_error.message + echo "exit code from processing is " $exitcode + exit $exitcode + fi # let's compare to previous step if [[ -f latest_reco_1.log ]]; then nCTFsFilesInspected_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/\(^[0-9]*\)_.*/\1/'` @@ -569,23 +594,29 @@ else exit $exitcode fi mv latest.log latest_reco_3.log + if [[ -f performanceMetrics.json ]]; then + mv performanceMetrics.json performanceMetrics_3.json + fi fi fi fi # now extract all performance metrics IFS=$'\n' -if [[ -f "performanceMetrics.json" ]]; then - timeStart=`date +%s` - for workflow in `grep ': {' performanceMetrics.json`; do - strippedWorkflow=`echo $workflow | cut -d\" -f2` - cat performanceMetrics.json | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics.json +timeStart=`date +%s` +for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json ; do + suffix=`echo $perfMetricsFiles | sed 's/performanceMetrics\(.*\).json/\1/'` + if [[ -f "performanceMetrics.json" ]]; then + for workflow in `grep ': {' $perfMetricsFiles`; do + strippedWorkflow=`echo $workflow | cut -d\" -f2` + cat $perfMetricsFiles | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics${suffix}.json done - timeEnd=`date +%s` - timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) - delta=$(( $timeEnd-$timeStart )) - echo "Time spent in splitting the metrics files = $delta s" -fi + fi +done +timeEnd=`date +%s` +timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) +delta=$(( $timeEnd-$timeStart )) +echo "Time spent in splitting the metrics files = $delta s" if [[ $ALIEN_JDL_AODOFF != 1 ]]; then # flag to possibly enable Analysis QC diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index c3bf1cdfd..8cdb8259f 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -240,7 +240,7 @@ fi echo "BeamType = $BEAMTYPE" -if [[ $ALIEN_JDL_ENABLEMONITORING == "1" ]]; then +if [[ $ALIEN_JDL_ENABLEMONITORING != "0" ]]; then # add the performance metrics export ENABLE_METRICS=1 export ARGS_ALL_EXTRA="$ARGS_ALL_EXTRA --resources-monitoring 50 --resources-monitoring-dump-interval 50" diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 21bf3b10e..9e55359af 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -39,18 +39,18 @@ N_TPCTRK=$NGPUS if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then # Tuned multiplicities for async processing if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_8cpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=3 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=3 NGPURECOTHREADS=5 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_16cpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=8 - [[ -z $SHMSIZE ]] && SHMSIZE=22000000000 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=22000000000 NGPURECOTHREADS=9 NTRDTRKTHREADS=3 ITSTRK_THREADS=3 ITSTPC_THREADS=3 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_64cpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=32 - [[ -z $SHMSIZE ]] && SHMSIZE=90000000000 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=32 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=90000000000 NGPURECOTHREADS=12 NTRDTRKTHREADS=3 ITSTRK_THREADS=3 @@ -63,8 +63,8 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_TOFMATCH=2 N_TPCENTDEC=3 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=8 - [[ -z $SHMSIZE ]] && SHMSIZE=20000000000 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=20000000000 N_TOFMATCH=2 N_MCHCL=3 N_TPCENTDEC=2 @@ -76,8 +76,8 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then ITSTRK_THREADS=2 ITSTPC_THREADS=2 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_4gpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=45 - [[ -z $SHMSIZE ]] && SHMSIZE=100000000000 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=45 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000 NGPURECOTHREADS=8 NTRDTRKTHREADS=2 ITSTRK_THREADS=2 @@ -98,9 +98,9 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_TPCITS=12 N_ITSTRK=12 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then - [[ -z $TIMEFRAME_RATE_LIMIT ]] && TIMEFRAME_RATE_LIMIT=20 - [[ -z $SHMSIZE ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4 - [[ -z $TIMEFRAME_SHM_LIMIT ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3)) + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=20 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4 + [[ -z ${TIMEFRAME_SHM_LIMIT:-} ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3)) NGPURECOTHREADS=8 NTRDTRKTHREADS=4 ITSTRK_THREADS=6 From e82a624970d247382763a5127403f8981e5340db Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Tue, 7 Nov 2023 14:07:52 +0100 Subject: [PATCH 0174/1239] [GenTest] Always include partonic event --- test/run_generator_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_generator_tests.sh b/test/run_generator_tests.sh index 5f91db447..bb256ba78 100755 --- a/test/run_generator_tests.sh +++ b/test/run_generator_tests.sh @@ -85,7 +85,7 @@ exec_test() echo "### Testing ${ini_path} with generator ${generator} ###" > ${LOG_FILE_GENERIC_KINE} echo "### Testing ${ini_path} with generator ${generator} ###" > ${LOG_FILE_SIM} # run the simulation, fail if not successful - o2-sim -g ${generator_lower} ${trigger} --noGeant -n 100 -j 4 --configFile ${ini_path} >> ${LOG_FILE_SIM} 2>&1 + o2-sim -g ${generator_lower} ${trigger} --noGeant -n 100 -j 4 --configFile ${ini_path} --configKeyValues "GeneratorPythia8.includePartonEvent=true" >> ${LOG_FILE_SIM} 2>&1 RET=${?} [[ "${RET}" != "0" ]] && { remove_artifacts ; return ${RET} ; } From 5245357b517bc45d7364b78addfc95bd0cd4c35d Mon Sep 17 00:00:00 2001 From: aimeric-landou <46970521+aimeric-landou@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:08:30 +0100 Subject: [PATCH 0175/1239] HF Jet MC - fix weighting, using new HF jet generator (#1319) --- .../ini/GeneratorHFJETrigger_ccbar.ini | 8 ++ .../ini/tests/GeneratorHFJETrigger_ccbar.C | 125 ++++++++++++++++++ .../pythia8_jet_charmtriggers_with_decays.cfg | 117 ++++++++++++++++ MC/run/PWGGAJE/run_jets_HF_ccbar.sh | 37 ++---- 4 files changed, 258 insertions(+), 29 deletions(-) create mode 100644 MC/config/PWGGAJE/ini/GeneratorHFJETrigger_ccbar.ini create mode 100644 MC/config/PWGGAJE/ini/tests/GeneratorHFJETrigger_ccbar.C create mode 100644 MC/config/PWGGAJE/pythia8/generator/pythia8_jet_charmtriggers_with_decays.cfg diff --git a/MC/config/PWGGAJE/ini/GeneratorHFJETrigger_ccbar.ini b/MC/config/PWGGAJE/ini/GeneratorHFJETrigger_ccbar.ini new file mode 100644 index 000000000..72b484652 --- /dev/null +++ b/MC/config/PWGGAJE/ini/GeneratorHFJETrigger_ccbar.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharm(3, -5, 5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGGAJE/pythia8/generator/pythia8_jet_charmtriggers_with_decays.cfg + diff --git a/MC/config/PWGGAJE/ini/tests/GeneratorHFJETrigger_ccbar.C b/MC/config/PWGGAJE/ini/tests/GeneratorHFJETrigger_ccbar.C new file mode 100644 index 000000000..ef24e0d3e --- /dev/null +++ b/MC/config/PWGGAJE/ini/tests/GeneratorHFJETrigger_ccbar.C @@ -0,0 +1,125 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + int checkPdgQuark{4}; + float ratioTrigger = 1. / 3; // one event triggered out of 3 + std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ + // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}}}, // Ds+ + {443, {{-11, 11}}}, // Jpsi + {4122, + {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, + {{-313, 2212}, + {-321, 3324}, + {211, 211, 3312}, + {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nQuarks{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + // Setting up event header to access event weight info + o2::dataformats::MCEventHeader *eventHeader = nullptr; + ; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + int weight1Counter = 0; + bool isvalid; + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + float eventWeight = eventHeader->getInfo("weight", isvalid); + if (!isvalid) { + std::cerr << "Could not retrieve event weight from MCEventHeader\n"; + return 1; + } + if (abs(eventWeight - 1) < + 1E-5) { // checks that the event weights are not unitary + weight1Counter++; + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuark) { + nQuarks++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), + std::abs(pdg)) != checkPdgHadron.end()) // found signal + { + // count signal PDG + nSignals++; + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; + j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuark) << nQuarks << "\n"; + std::cout << "# signal hadrons: " << nSignals << "\n"; + std::cout << "# signal hadrons decaying in the correct channel: " + << nSignalGoodDecay << "\n"; + + if (nQuarks < + 2 * nEvents * + ratioTrigger) // we expect anyway more because the same quark is + // repeated several time, after each gluon radiation + { + std::cerr << "Number of generated (anti)quarks " << checkPdgQuark + << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) // we put some tolerance (e.g. due to oscillations + // which might change the final state) + { + std::cerr << "Fraction of signals decaying into the correct channel " + << fracForcedDecays << " lower than expected\n"; + return 1; + } + + if (weight1Counter == nEvents) { + std::cerr << "All events have a unitary weight\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGGAJE/pythia8/generator/pythia8_jet_charmtriggers_with_decays.cfg b/MC/config/PWGGAJE/pythia8/generator/pythia8_jet_charmtriggers_with_decays.cfg new file mode 100644 index 000000000..9c0089c1c --- /dev/null +++ b/MC/config/PWGGAJE/pythia8/generator/pythia8_jet_charmtriggers_with_decays.cfg @@ -0,0 +1,117 @@ +### author: Aimeric Landou (aimeric.landou@cern.ch) +### based on Fabrizio Grosa's HF template (fabrizio.grosa@cern.ch) +### since: January 2023 + +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +# ### processes +# SoftQCD:inelastic on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### Force golden charm hadrons decay modes for trigger studies + +### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other +411:oneChannel = 1 0.0752 0 -321 211 211 +411:addChannel = 1 0.0104 0 -313 211 +411:addChannel = 1 0.0156 0 311 211 +411:addChannel = 1 0.00276 0 333 211 +## add Lc decays absent in PYTHIA8 decay table and set BRs from PDG for other +4122:oneChannel = 1 0.0196 100 2212 -313 +4122:addChannel = 1 0.0108 100 2224 -321 +4122:addChannel = 1 0.022 100 3124 211 +4122:addChannel = 1 0.035 0 2212 -321 211 +### add Xic+ decays absent in PYTHIA8 decay table +4232:addChannel = 1 0.2 0 2212 -313 +4232:addChannel = 1 0.2 0 2212 -321 211 +4232:addChannel = 1 0.2 0 3324 211 +4232:addChannel = 1 0.2 0 3312 211 211 +### add Xic0 decays absent in PYTHIA8 decay table +4132:addChannel = 1 0.2 0 3312 211 + +### K* -> K pi +313:onMode = off +313:onIfAll = 321 211 +### for Ds -> Phi pi+ +333:onMode = off +333:onIfAll = 321 321 +### for D0 -> rho0 pi+ k- +113:onMode = off +113:onIfAll = 211 211 +### for Lambda_c -> Delta++ K- +2224:onMode = off +2224:onIfAll = 2212 211 +### for Lambda_c -> Lambda(1520) K- +102134:onMode = off +102134:onIfAll = 2212 321 + +### switch off all decay channels +411:onMode = off +421:onMode = off +431:onMode = off +4112:onMode = off +4122:onMode = off +4232:onMode = off +4132:onMode = off +443:onMode = off +4332:onMode = off + +### D0 -> K pi +421:onIfMatch = 321 211 + +### D+/- -> K pi pi +411:onIfMatch = 321 211 211 +### D+/- -> K* pi +411:onIfMatch = 313 211 +### D+/- -> phi pi +411:onIfMatch = 333 211 + +### D_s -> Phi pi +431:onIfMatch = 333 211 + +### Lambda_c -> p K* +4122:onIfMatch = 2212 313 +### Lambda_c -> Delta K +4122:onIfMatch = 2224 321 +### Lambda_c -> Lambda(1520) pi +4122:onIfMatch = 3124 211 +### Lambda_c -> p K pi +4122:onIfMatch = 2212 321 211 + +### Xic+ -> pK*0 +4232:onIfMatch = 2212 313 +### Xic+ -> p K- pi+ +4232:onIfMatch = 2212 321 211 +### Xic+ -> Xi*0 pi+, Xi*->Xi- pi+ +4232:onIfMatch = 3324 211 +### Xic+ -> Xi- pi+ pi+ +4232:onIfMatch = 3312 211 211 + +### Xic0 -> Xi- pi+ +4132:onIfMatch = 3312 211 + +### Omega_c -> Omega pi +4332:onIfMatch = 3334 211 + +### Jpsi -> ee +443:onIfMatch = 11 11 + +# Correct Lb decay length (wrong in PYTHIA8 decay table) +5122:tau0 = 4.41000e-01 + +# Correct OmegaC decay length (wrong in PYTHIA8 decay table) +4332:tau0 = 0.08000000000 + + +### Jets options +PhaseSpace:pTHatMin = 5.0 +PhaseSpace:pTHatMax = 300.0 +PhaseSpace:bias2Selection = on +PhaseSpace:bias2SelectionPow = 6.0 +SoftQCD:inelastic off # all inelastic processes +HardQCD:all on \ No newline at end of file diff --git a/MC/run/PWGGAJE/run_jets_HF_ccbar.sh b/MC/run/PWGGAJE/run_jets_HF_ccbar.sh index 3149f5092..9b155ba2b 100644 --- a/MC/run/PWGGAJE/run_jets_HF_ccbar.sh +++ b/MC/run/PWGGAJE/run_jets_HF_ccbar.sh @@ -15,42 +15,21 @@ RNDSEED=${RNDSEED:-0} # [default = 0] time-based random seed -NSIGEVENTS=${NSIGEVENTS:-10} +NSIGEVENTS=${NSIGEVENTS:-5} NTIMEFRAMES=${NTIMEFRAMES:-1} NWORKERS=${NWORKERS:-8} MODULES="--skipModules ZDC" #"PIPE ITS TPC EMCAL" CONFIG_ENERGY=${CONFIG_ENERGY:-13600.0} SIMENGINE=${SIMENGINE:-TGeant4} -WEIGHTPOW=${WEIGHTPOW:-6.0} [[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" -# Default for weighted productions -PTHATMIN=${PTHATMIN:-5.0} -PTHATMAX=${PTHATMAX:-300.0} - -# Define the pt hat bin arrays -pthatbin_loweredges=(0 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235) -pthatbin_higheredges=( 5 7 9 12 16 21 28 36 45 57 70 85 99 115 132 150 169 190 212 235 -1) - -# Recover environmental vars for pt binning -#PTHATBIN=${PTHATBIN:-1} - -if [ -z "$PTHATBIN" ]; then - echo "Open Pt-hat range set" -else - PTHATMIN=${pthatbin_loweredges[$PTHATBIN]} - PTHATMAX=${pthatbin_higheredges[$PTHATBIN]} -fi - - -#ccbar filter -${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${CONFIG_ENERGY} -col pp -gen external -proc "jets" \ - -ptHatMin ${PTHATMIN} -ptHatMax ${PTHATMAX} \ - -tf ${NTIMEFRAMES} -ns ${NSIGEVENTS} -e ${SIMENGINE} \ - -j ${NWORKERS} -mod "--skipModules ZDC" \ - -interactionRate 500000 -confKey "Diamond.width[2]=6." ${SEED} \ - -ini $O2DPG_ROOT/MC/config/PWGHF/ini/GeneratorHFTrigger_ccbar.ini \ - -weightPow ${WEIGHTPOW} +#ccbar filter and bias2SelectionPow and PtHat settings are in the ini file given below +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${CONFIG_ENERGY} -col pp -gen external -proc "jets" \ + -tf ${NTIMEFRAMES} -ns ${NSIGEVENTS} -e ${SIMENGINE} \ + -j ${NWORKERS} -mod "--skipModules ZDC" \ + -interactionRate 500000 -confKey "Diamond.width[2]=6." ${SEED} \ + -ini $O2DPG_ROOT/MC/config/PWGGAJE/ini/GeneratorHFJETrigger_ccbar.ini + # run workflow # allow increased timeframe parallelism with --cpu-limit 32 From 1c2955e8fe05127d87f7fe263a064cdacd40327b Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Wed, 8 Nov 2023 14:31:22 +0100 Subject: [PATCH 0176/1239] Add cluster output to laser (#1309) --- DATA/production/calib/tpc-laser-filter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index fb3c0cf3d..2315e245c 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -66,7 +66,7 @@ o2-dpl-raw-proxy $ARGS_ALL \ --send-ce-digits \ | o2-tpc-reco-workflow $ARGS_ALL ${TPC_CORR_SCALING:-} \ --input-type digitizer \ - --output-type "tracks,disable-writer" \ + --output-type "tracks,disable-writer,clusters" \ --disable-mc \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ $GPU_CONFIG \ From ec4b1b728c69bf4dd7bea6b854417e10b40b40e1 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Thu, 2 Nov 2023 16:45:34 +0100 Subject: [PATCH 0177/1239] Keep parton-level information --- MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini | 1 + MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini | 1 + MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini index 1f04a3ade..e141807e0 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini @@ -5,3 +5,4 @@ funcName=GeneratorPythia8GapTriggeredBeauty(3, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini index 31ec8e82d..4a1947cf1 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini @@ -5,3 +5,4 @@ funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini index c5e668f9f..c159fdc6e 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini @@ -5,3 +5,4 @@ funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +includePartonEvent=true From 2eb733308867de1228fbec8972824a9660f6780c Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Thu, 9 Nov 2023 17:04:33 +0100 Subject: [PATCH 0178/1239] Cell write does not need config values fixes an unknown key crash seen on ubuntu --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 9db08a754..ea4340c5f 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1054,7 +1054,7 @@ def getDigiTaskName(det): EMCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] EMCRECOtask['cmd'] += ' | ${O2_ROOT}/bin/o2-emcal-cell-recalibrator-workflow --input-subspec 1 --output-subspec 0 --no-timecalib --no-gaincalib ' + putConfigValues() EMCRECOtask['cmd'] += (' --isMC','')[args.no_mc_labels] - EMCRECOtask['cmd'] += ' | ${O2_ROOT}/bin/o2-emcal-cell-writer-workflow --subspec 0 ' + getDPL_global_options() + putConfigValues() + EMCRECOtask['cmd'] += ' | ${O2_ROOT}/bin/o2-emcal-cell-writer-workflow --subspec 0 ' + getDPL_global_options() EMCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(EMCRECOtask) From 36ab46dcbec329056a59e468bb77f36d606b1667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Fri, 10 Nov 2023 15:51:11 +0100 Subject: [PATCH 0179/1239] dpgsim with compatibility with old O2 (#1291) --- MC/bin/o2dpg_sim_workflow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index ea4340c5f..72f9d1f12 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1312,7 +1312,10 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] # strangeness tracking is now called from the secondary vertexer if not args.with_strangeness_tracking: - SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' + from subprocess import run + data = run(SVFINDERtask['cmd'].split(" ")[0] + " --help", capture_output=True, shell=True, text=True) + if "disable-strangeness-tracker" in data.stdout: + SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' # if enabled, it may require MC labels else: SVFINDERtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] From f5c961c4d5dc1450eb5110a7d986440bcef34ee1 Mon Sep 17 00:00:00 2001 From: ZFederica Date: Fri, 10 Nov 2023 17:11:42 +0100 Subject: [PATCH 0180/1239] Change generator setting --- MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini | 2 +- MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini b/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini index d38c3b4f3..7c09045ee 100644 --- a/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini +++ b/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4332,true) +funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4332) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini b/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini index 11c1ab205..d5915c399 100644 --- a/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini +++ b/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4132,true) +funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4132) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg From 029090d5bb0f9e841c87989c99fb1ac7ed80f814 Mon Sep 17 00:00:00 2001 From: ZFederica Date: Fri, 10 Nov 2023 17:43:42 +0100 Subject: [PATCH 0181/1239] Switch to mode 2 --- .../pythia8_charmtriggers_omegactoxipi.cfg | 18 ++++++++++++++++++ .../pythia8_charmtriggers_xictoxipi.cfg | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg index 1c72afe9b..198d11cfe 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg @@ -15,6 +15,24 @@ ParticleDecays:tau0Max 100. # Correct OmegaC decay length (wrong in PYTHIA8 decay table) (mm/c) 4332:tau0 = 0.08000000000 +### switching on Pythia Mode2 +ColourReconnection:mode 1 +ColourReconnection:allowDoubleJunRem off +ColourReconnection:m0 0.3 +ColourReconnection:allowJunctions on +ColourReconnection:junctionCorrection 1.20 +ColourReconnection:timeDilationMode 2 +ColourReconnection:timeDilationPar 0.18 +StringPT:sigma 0.335 +StringZ:aLund 0.36 +StringZ:bLund 0.56 +StringFlav:probQQtoQ 0.078 +StringFlav:ProbStoUD 0.2 +StringFlav:probQQ1toQQ0join 0.0275,0.0275,0.0275,0.0275 +MultiPartonInteractions:pT0Ref 2.15 +BeamRemnants:remnantMode 1 +BeamRemnants:saturation 5 + ### add OmegaC decay absent in PYTHIA8 decay table 4332:addChannel = 1 0.02 0 3312 211 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg index bed2ee9ee..e19ce491b 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg @@ -14,6 +14,24 @@ ParticleDecays:tau0Max 100. # Correct XiC decay length (mm/c) 4132:tau0 = 0.04557000000 +### switching on Pythia Mode2 +ColourReconnection:mode 1 +ColourReconnection:allowDoubleJunRem off +ColourReconnection:m0 0.3 +ColourReconnection:allowJunctions on +ColourReconnection:junctionCorrection 1.20 +ColourReconnection:timeDilationMode 2 +ColourReconnection:timeDilationPar 0.18 +StringPT:sigma 0.335 +StringZ:aLund 0.36 +StringZ:bLund 0.56 +StringFlav:probQQtoQ 0.078 +StringFlav:ProbStoUD 0.2 +StringFlav:probQQ1toQQ0join 0.0275,0.0275,0.0275,0.0275 +MultiPartonInteractions:pT0Ref 2.15 +BeamRemnants:remnantMode 1 +BeamRemnants:saturation 5 + ### add XiC decay absent in PYTHIA8 decay table 4132:addChannel = 1 0.02 0 3312 211 From 9f31f78713c1886de51620fde8da1031e40e2fef Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 10 Nov 2023 10:16:39 +0100 Subject: [PATCH 0182/1239] Add --inject-missing-data to all detector workflows receiving data from TfBuilder --- DATA/production/calib/hmp-pedestals-processing.sh | 2 +- DATA/production/calib/its-noise-processing.sh | 2 +- DATA/production/calib/its-threshold-processing.sh | 2 +- DATA/production/calib/mch-badchannel-processing.sh | 2 +- DATA/production/calib/mft-noise-processing.sh | 2 +- DATA/production/calib/phs-led.sh | 2 +- DATA/production/calib/phs-pedestal.sh | 2 +- DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh | 2 +- DATA/production/calib/tof-standalone-reco.sh | 2 +- DATA/production/calib/tpc-laser-filter.sh | 2 +- DATA/production/calib/tpc-laser.sh | 2 +- DATA/production/calib/tpc-pedestal.sh | 2 +- DATA/production/calib/tpc-pulser-long.sh | 2 +- DATA/production/calib/tpc-pulser.sh | 2 +- DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPileline.sh | 2 +- DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineCTF.sh | 2 +- .../detectors/EMC/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh | 2 +- .../detectors/EMC/runEMCRawToDigitsRecoPilelineQClocal.sh | 2 +- .../detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTF.sh | 2 +- .../EMC/runEMCRawToDigitsRecoPilelineQClocalCTFSingle.sh | 2 +- DATA/testing/detectors/FDD/fdd-ctf.sh | 2 +- DATA/testing/detectors/FDD/fdd-digits-ctf.sh | 2 +- DATA/testing/detectors/FDD/fdd-digits-qc-ctf.sh | 2 +- DATA/testing/detectors/FT0/ft0-ctf.sh | 2 +- DATA/testing/detectors/FV0/fv0-ctf.sh | 2 +- DATA/testing/detectors/FV0/fv0-digits-ctf.sh | 2 +- DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh | 2 +- DATA/testing/detectors/MID/mid-calib-workflow.sh | 2 +- DATA/testing/detectors/MID/mid-raw-to-ctf-qc-workflow.sh | 2 +- DATA/testing/detectors/TOF/tof-epn-cosmics-dig.sh | 2 +- DATA/testing/detectors/TOF/tof-epn-cosmics-digNoQC.sh | 2 +- DATA/testing/detectors/TOF/tof-epn-cosmics.sh | 2 +- DATA/testing/detectors/TOF/tof-epn-cosmicsNoQC.sh | 2 +- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 2 +- DATA/testing/detectors/TPC/tpc-krypton.sh | 2 +- DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh | 2 +- DATA/testing/detectors/TPC/tpc-pattern-generator.sh | 2 +- DATA/testing/detectors/TRD/trd-krypton.sh | 2 +- DATA/testing/detectors/TRD/trd-noise.sh | 2 +- DATA/testing/examples/example-calib-processing.sh | 2 +- DATA/testing/examples/example-workflow.sh | 2 +- DATA/testing/private/afurs/fdd-ft0-fv0-ctf.sh | 2 +- DATA/testing/private/afurs/fdd-ft0-fv0-digits-qc-ctf.sh | 2 +- DATA/testing/private/afurs/ft0-fv0-ctf.sh | 2 +- DATA/testing/private/afurs/ft0-fv0-digits-qc-ctf.sh | 2 +- .../mfasel/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh | 2 +- DATA/testing/private/zampolli/zampolli-workflow.sh | 2 +- 47 files changed, 47 insertions(+), 47 deletions(-) diff --git a/DATA/production/calib/hmp-pedestals-processing.sh b/DATA/production/calib/hmp-pedestals-processing.sh index 6c05350a4..f7c48d737 100755 --- a/DATA/production/calib/hmp-pedestals-processing.sh +++ b/DATA/production/calib/hmp-pedestals-processing.sh @@ -74,7 +74,7 @@ SPEC_PARAM+="--pedestals-tag ${HMP_PED_TAG} --sigmacut ${HMP_SIGMACUT}" # Here we compose the workflow # Start with an empty workflow WORKFLOW= -add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\"" "" 0 +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=1,transport=shmem\"" "" 0 add_W o2-hmpid-raw-to-pedestals-workflow "--fast-decode $SPEC_PARAM" # Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) diff --git a/DATA/production/calib/its-noise-processing.sh b/DATA/production/calib/its-noise-processing.sh index 3d0181a14..fdd8b62c8 100755 --- a/DATA/production/calib/its-noise-processing.sh +++ b/DATA/production/calib/its-noise-processing.sh @@ -20,7 +20,7 @@ fi [[ -z $NITSDECTPIPELINES ]] && NITSDECTPIPELINES=6 WORKFLOW= -add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1\"" "" 0 add_W o2-itsmft-stf-decoder-workflow "${OUTTYPE} --nthreads ${NITSDECTHREADS} --pipeline its-stf-decoder:${NITSDECTPIPELINES} --ignore-noise-map" add_W o2-dpl-output-proxy "--dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-noise-input-proxy --channel-config \"name=its-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=1\"" "" 0 WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" diff --git a/DATA/production/calib/its-threshold-processing.sh b/DATA/production/calib/its-threshold-processing.sh index 8a768f8c1..e40d1845b 100755 --- a/DATA/production/calib/its-threshold-processing.sh +++ b/DATA/production/calib/its-threshold-processing.sh @@ -36,7 +36,7 @@ if [ $RUNTYPE_ITS == "totfullfast" ]; then fi WORKFLOW= -add_W o2-dpl-raw-proxy "--exit-transition-timeout 20 --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=0,transport=shmem\"" "" 0 +add_W o2-dpl-raw-proxy "--exit-transition-timeout 20 --dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,rateLogging=0,transport=shmem\"" "" 0 add_W o2-itsmft-stf-decoder-workflow "${ADDITIONAL_OPTIONS_DEC} --condition-tf-per-query -1 --condition-backend \"http://localhost:8084\" --ignore-dist-stf --nthreads 1 --no-clusters --no-cluster-patterns --pipeline its-stf-decoder:${NDECODERS} --enable-calib-data --digits" for i in $(seq 0 $((CHIPMODBASE-1))) do diff --git a/DATA/production/calib/mch-badchannel-processing.sh b/DATA/production/calib/mch-badchannel-processing.sh index cb1227e1f..771aeab87 100755 --- a/DATA/production/calib/mch-badchannel-processing.sh +++ b/DATA/production/calib/mch-badchannel-processing.sh @@ -10,7 +10,7 @@ source common/getCommonArgs.sh PROXY_INSPEC="A:MCH/RAWDATA;B:FLP/DISTSUBTIMEFRAME/0" PROXY_OUTSPEC="downstream:MCH/PDIGITS/0" -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " +WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " WORKFLOW+="o2-mch-pedestal-decoding-workflow --pipeline mch-pedestal-decoder:${MULTIPLICITY_FACTOR_RAWDECODERS} --logging-interval 10 ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" | " WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name mch-badchannel-input-proxy --channel-config \"name=mch-badchannel-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" diff --git a/DATA/production/calib/mft-noise-processing.sh b/DATA/production/calib/mft-noise-processing.sh index 4a8491844..cbcc52de8 100755 --- a/DATA/production/calib/mft-noise-processing.sh +++ b/DATA/production/calib/mft-noise-processing.sh @@ -9,7 +9,7 @@ source common/getCommonArgs.sh PROXY_INSPEC="A:MFT/RAWDATA;B:FLP/DISTSUBTIMEFRAME/0" PROXY_OUTSPEC="downstream:MFT/DIGITS/0;downstream:MFT/DIGITSROF/0" -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " +WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " WORKFLOW+="o2-itsmft-stf-decoder-workflow ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --runmft --digits --no-clusters --no-cluster-patterns --ignore-noise-map --nthreads 5 | " WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name mft-noise-input-proxy --channel-config \"name=mft-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" diff --git a/DATA/production/calib/phs-led.sh b/DATA/production/calib/phs-led.sh index a838f3c5c..b92e78b92 100755 --- a/DATA/production/calib/phs-led.sh +++ b/DATA/production/calib/phs-led.sh @@ -19,7 +19,7 @@ fi QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-led-qc o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-phos-reco-workflow $ARGS_ALL \ --input-type raw \ diff --git a/DATA/production/calib/phs-pedestal.sh b/DATA/production/calib/phs-pedestal.sh index a7a7e8f6d..7c285a8f6 100755 --- a/DATA/production/calib/phs-pedestal.sh +++ b/DATA/production/calib/phs-pedestal.sh @@ -19,7 +19,7 @@ fi QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-pedestal-qc o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-phos-reco-workflow $ARGS_ALL \ --input-type raw \ diff --git a/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh b/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh index 04d35cc29..5c96e7c7b 100755 --- a/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh +++ b/DATA/production/calib/tof-standalone-cosmic-reco-time-calib.sh @@ -13,7 +13,7 @@ PROXY_OUTSPEC="calclus:TOF/INFOCALCLUS;cosmics:TOF/INFOCOSMICS;trkcos:TOF/INFOTR MYDIR="$(dirname $(readlink -f $0))/../../testing/detectors/TOF" WORKFLOW= -add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " +add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " add_W o2-tof-reco-workflow "--input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" " add_W o2-qc "${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn " add_W o2-dpl-output-proxy "${ARGS_ALL} --dataspec ${PROXY_OUTSPEC} --proxy-channel-name tof-time-calib-input-proxy --channel-config \"name=tof-time-calib-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " diff --git a/DATA/production/calib/tof-standalone-reco.sh b/DATA/production/calib/tof-standalone-reco.sh index bd14b936e..67d3700ed 100755 --- a/DATA/production/calib/tof-standalone-reco.sh +++ b/DATA/production/calib/tof-standalone-reco.sh @@ -13,7 +13,7 @@ PROXY_OUTSPEC="diagWords:TOF/DIAFREQ" MYDIR="$(dirname $(readlink -f $0))/../../testing/detectors/TOF" WORKFLOW= -add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " +add_W o2-dpl-raw-proxy "${ARGS_ALL} --dataspec ${PROXY_INSPEC} --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" " add_W o2-tof-reco-workflow "--input-type raw --output-type clusters ${ARGS_ALL} --configKeyValues ${ARGS_ALL_CONFIG} --disable-root-output --calib-cluster --cluster-time-window 10000 --cosmics --pipeline \"tof-compressed-decoder:${NTHREADS},TOFClusterer:${NTHREADS}\" " add_W o2-qc "${ARGS_ALL} --config json://${MYDIR}/qc-full.json --local --host epn " add_W o2-dpl-output-proxy "${ARGS_ALL} --dataspec ${PROXY_OUTSPEC} --proxy-channel-name tof-diagn-input-proxy --channel-config \"name=tof-diagn-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" " diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 2315e245c..677c14f90 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -55,7 +55,7 @@ if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then fi o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL ${LASER_DECODER_ADD} \ --input-spec "$CALIB_INSPEC" \ diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index 76ac79ec0..83d616895 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -84,7 +84,7 @@ if [[ ${TPC_CALIB_TRACKS_PUBLISH_EOS:-} == 1 ]]; then fi -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL ${LASER_DECODER_ADD}\ diff --git a/DATA/production/calib/tpc-pedestal.sh b/DATA/production/calib/tpc-pedestal.sh index 31039db4f..80e2f003c 100755 --- a/DATA/production/calib/tpc-pedestal.sh +++ b/DATA/production/calib/tpc-pedestal.sh @@ -37,7 +37,7 @@ EXTRA_CONFIG=" --publish-after-tfs ${publish_after} --max-events ${max_events} - ################################################################################################################################# -o2-dpl-raw-proxy ${ARGS_ALL} \ +o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ --dataspec "${PROXY_INSPEC}" \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-tpc-calib-pad-raw ${ARGS_ALL} \ diff --git a/DATA/production/calib/tpc-pulser-long.sh b/DATA/production/calib/tpc-pulser-long.sh index 91c9959b9..8cd2c38e9 100755 --- a/DATA/production/calib/tpc-pulser-long.sh +++ b/DATA/production/calib/tpc-pulser-long.sh @@ -36,7 +36,7 @@ EXTRA_CONFIG="--calib-type pulser --reset-after-publish --publish-after-tfs ${pu ################################################################################################################################# -o2-dpl-raw-proxy ${ARGS_ALL} \ +o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ --dataspec ${PROXY_INSPEC} \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-tpc-calib-pad-raw ${ARGS_ALL} \ diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index 9541e42ca..f7934e352 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -38,7 +38,7 @@ EXTRA_CONFIG="--calib-type pulser --publish-after-tfs ${publish_after} --max-eve ################################################################################################################################# -o2-dpl-raw-proxy ${ARGS_ALL} \ +o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ --dataspec "${PROXY_INSPEC}" \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-tpc-calib-pad-raw ${ARGS_ALL} \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPileline.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPileline.sh index 0e8eb075f..11b82101b 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPileline.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPileline.sh @@ -11,7 +11,7 @@ SEVERITY_RAWPROXY=warning INFOLOGGER_SEVERITY=warning SEVERITY=warning -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $INFOLOGGER_SEVERITY_RAWPROXY \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineCTF.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineCTF.sh index f90949474..42894d213 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineCTF.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineCTF.sh @@ -21,7 +21,7 @@ CTF_DICT="${FILEWORKDIR}/ctf_dictionary.root" CTF_MINSIZE="500000000" CTF_MAX_PER_FILE=10000 -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity info \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh index 0cb6ea090..f23b66842 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh @@ -24,7 +24,7 @@ CTF_DICT="${FILEWORKDIR}/ctf_dictionary.root" CTF_MINSIZE="500000000" CTF_MAX_PER_FILE=10000 -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $SEVERITY_RAWPROXY \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocal.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocal.sh index 3bb2d27cb..ee970b148 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocal.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocal.sh @@ -16,7 +16,7 @@ SEVERITY_RAWPROXY=warning INFOLOGGER_SEVERITY=warning SEVERITY=warning -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $SEVERITY_RAWPROXY \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTF.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTF.sh index 6f7f49fe4..8a085b729 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTF.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTF.sh @@ -24,7 +24,7 @@ CTF_DICT="${FILEWORKDIR}/ctf_dictionary.root" CTF_MINSIZE="500000000" CTF_MAX_PER_FILE=10000 -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $SEVERITY_RAWPROXY \ diff --git a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTFSingle.sh b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTFSingle.sh index 19a5fbc25..88c5b260d 100755 --- a/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTFSingle.sh +++ b/DATA/testing/detectors/EMC/runEMCRawToDigitsRecoPilelineQClocalCTFSingle.sh @@ -22,7 +22,7 @@ CTF_DICT="${FILEWORKDIR}/ctf_dictionary.root" CTF_MINSIZE="500000000" CTF_MAX_PER_FILE=10000 -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy $ARGS_ALL --inject-missing-data \ --dataspec "$PROXY_INSPEC" \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $SEVERITY_RAWPROXY \ diff --git a/DATA/testing/detectors/FDD/fdd-ctf.sh b/DATA/testing/detectors/FDD/fdd-ctf.sh index 1f4c51ded..e3e943feb 100755 --- a/DATA/testing/detectors/FDD/fdd-ctf.sh +++ b/DATA/testing/detectors/FDD/fdd-ctf.sh @@ -15,7 +15,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="digits:FDD/DIGITSBC/0;channels:FDD/DIGITSCH/0;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fdd-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FDD --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ | o2-dpl-run $ARGS_ALL $GLOBALDPLOPT --dds ${WORKFLOWMODE_FILE} # option instead iof run to export DDS xml file diff --git a/DATA/testing/detectors/FDD/fdd-digits-ctf.sh b/DATA/testing/detectors/FDD/fdd-digits-ctf.sh index c27259d0c..a13d56942 100755 --- a/DATA/testing/detectors/FDD/fdd-digits-ctf.sh +++ b/DATA/testing/detectors/FDD/fdd-digits-ctf.sh @@ -15,7 +15,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FDD/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fdd-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fdd-datareader-dpl:$NTHREADS \ | o2-fdd-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FDD --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ diff --git a/DATA/testing/detectors/FDD/fdd-digits-qc-ctf.sh b/DATA/testing/detectors/FDD/fdd-digits-qc-ctf.sh index af66422a5..0e109927b 100755 --- a/DATA/testing/detectors/FDD/fdd-digits-qc-ctf.sh +++ b/DATA/testing/detectors/FDD/fdd-digits-qc-ctf.sh @@ -15,7 +15,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FDD/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fdd-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fdd-datareader-dpl:$NTHREADS \ | o2-fdd-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-qc ${ARGS_ALL} --local --host epn --config json://${MYDIR}/fdd-digits-ds.json \ diff --git a/DATA/testing/detectors/FT0/ft0-ctf.sh b/DATA/testing/detectors/FT0/ft0-ctf.sh index e0a4fbc09..91ea21b33 100755 --- a/DATA/testing/detectors/FT0/ft0-ctf.sh +++ b/DATA/testing/detectors/FT0/ft0-ctf.sh @@ -16,7 +16,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="digits:FT0/DIGITSBC/0;channels:FT0/DIGITSCH/0;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-ft0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FT0 --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ | o2-dpl-run $ARGS_ALL $GLOBALDPLOPT --dds ${WORKFLOWMODE_FILE} # option instead iof run to export DDS xml file diff --git a/DATA/testing/detectors/FV0/fv0-ctf.sh b/DATA/testing/detectors/FV0/fv0-ctf.sh index 0fe721a33..c00a5932f 100755 --- a/DATA/testing/detectors/FV0/fv0-ctf.sh +++ b/DATA/testing/detectors/FV0/fv0-ctf.sh @@ -15,7 +15,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="digits:FV0/DIGITSBC/0;channels:FV0/DIGITSCH/0;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FV0 --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ | o2-dpl-run $ARGS_ALL $GLOBALDPLOPT --dds ${WORKFLOWMODE_FILE} # option instead iof run to export DDS xml file diff --git a/DATA/testing/detectors/FV0/fv0-digits-ctf.sh b/DATA/testing/detectors/FV0/fv0-digits-ctf.sh index 4f5141d49..08ccb31e0 100755 --- a/DATA/testing/detectors/FV0/fv0-digits-ctf.sh +++ b/DATA/testing/detectors/FV0/fv0-digits-ctf.sh @@ -15,7 +15,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FV0/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fv0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fv0-datareader-dpl:$NTHREADS \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FV0 --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ diff --git a/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh b/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh index 62a5accd9..72f226e31 100755 --- a/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh +++ b/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh @@ -18,7 +18,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FV0/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fv0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fv0-datareader-dpl:$NTHREADS \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-qc ${ARGS_ALL} --local --host epn --config json://${MYDIR}/fv0-digits-ds.json \ diff --git a/DATA/testing/detectors/MID/mid-calib-workflow.sh b/DATA/testing/detectors/MID/mid-calib-workflow.sh index 7ddbafe5d..e846ea94f 100755 --- a/DATA/testing/detectors/MID/mid-calib-workflow.sh +++ b/DATA/testing/detectors/MID/mid-calib-workflow.sh @@ -3,7 +3,7 @@ # shellcheck disable=SC1091 source testing/detectors/MID/mid_common.sh -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --channel-config \"$MID_DPL_CHANNEL_CONFIG\" | " +WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --inject-missing-data --channel-config \"$MID_DPL_CHANNEL_CONFIG\" | " WORKFLOW+="o2-mid-raw-to-digits-workflow $ARGS_ALL $MID_RAW_TO_DIGITS_OPTS | " WORKFLOW+="o2-mid-calibration-workflow $ARGS_ALL | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " diff --git a/DATA/testing/detectors/MID/mid-raw-to-ctf-qc-workflow.sh b/DATA/testing/detectors/MID/mid-raw-to-ctf-qc-workflow.sh index aa9919102..d731fb767 100755 --- a/DATA/testing/detectors/MID/mid-raw-to-ctf-qc-workflow.sh +++ b/DATA/testing/detectors/MID/mid-raw-to-ctf-qc-workflow.sh @@ -3,7 +3,7 @@ # shellcheck disable=SC1091 source testing/detectors/MID/mid_common.sh -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --channel-config \"$MID_DPL_CHANNEL_CONFIG\" | " +WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --inject-missing-data --channel-config \"$MID_DPL_CHANNEL_CONFIG\" | " WORKFLOW+="o2-mid-raw-to-digits-workflow $ARGS_ALL $MID_RAW_TO_DIGITS_OPTS | " WORKFLOW+="o2-mid-entropy-encoder-workflow $ARGS_ALL | " WORKFLOW+="o2-ctf-writer-workflow $ARGS_ALL $MID_CTF_WRITER_OPTS | " diff --git a/DATA/testing/detectors/TOF/tof-epn-cosmics-dig.sh b/DATA/testing/detectors/TOF/tof-epn-cosmics-dig.sh index b8781db74..d96bae672 100755 --- a/DATA/testing/detectors/TOF/tof-epn-cosmics-dig.sh +++ b/DATA/testing/detectors/TOF/tof-epn-cosmics-dig.sh @@ -17,7 +17,7 @@ OUT_CHANNEL="name=downstream,method=connect,address=tcp://${calibration_node},ty PROXY_OUTSPEC="dd:FLP/DISTSUBTIMEFRAME/0;dig:TOF/DIGITS;head:TOF/DIGITHEADER;row:TOF/READOUTWINDOW;patt:TOF/PATTERNS;diafreq:TOF/DIAFREQ" -o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tof-reco-workflow --input-type raw --output-type digits --disable-root-output \ ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" \ diff --git a/DATA/testing/detectors/TOF/tof-epn-cosmics-digNoQC.sh b/DATA/testing/detectors/TOF/tof-epn-cosmics-digNoQC.sh index d50e2c8e0..3546e5e23 100755 --- a/DATA/testing/detectors/TOF/tof-epn-cosmics-digNoQC.sh +++ b/DATA/testing/detectors/TOF/tof-epn-cosmics-digNoQC.sh @@ -17,7 +17,7 @@ OUT_CHANNEL="name=downstream,method=connect,address=tcp://${calibration_node},ty PROXY_OUTSPEC="dd:FLP/DISTSUBTIMEFRAME/0;dig:TOF/DIGITS;head:TOF/DIGITHEADER;row:TOF/READOUTWINDOW;patt:TOF/PATTERNS;diafreq:TOF/DIAFREQ" -o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tof-reco-workflow --input-type raw --output-type digits --disable-root-output \ ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" \ diff --git a/DATA/testing/detectors/TOF/tof-epn-cosmics.sh b/DATA/testing/detectors/TOF/tof-epn-cosmics.sh index 32f489fd7..fbe7c4259 100755 --- a/DATA/testing/detectors/TOF/tof-epn-cosmics.sh +++ b/DATA/testing/detectors/TOF/tof-epn-cosmics.sh @@ -17,7 +17,7 @@ OUT_CHANNEL="name=downstream,method=connect,address=tcp://${calibration_node},ty PROXY_OUTSPEC="dd:FLP/DISTSUBTIMEFRAME/0;calclus:TOF/INFOCALCLUS;cosmics:TOF/INFOCOSMICS;trkcos:TOF/INFOTRACKCOS;trksiz:TOF/INFOTRACKSIZE" -o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tof-reco-workflow --input-type raw --output-type clusters \ ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" \ diff --git a/DATA/testing/detectors/TOF/tof-epn-cosmicsNoQC.sh b/DATA/testing/detectors/TOF/tof-epn-cosmicsNoQC.sh index 24d37a8e2..28ece2314 100755 --- a/DATA/testing/detectors/TOF/tof-epn-cosmicsNoQC.sh +++ b/DATA/testing/detectors/TOF/tof-epn-cosmicsNoQC.sh @@ -17,7 +17,7 @@ OUT_CHANNEL="name=downstream,method=connect,address=tcp://${calibration_node},ty PROXY_OUTSPEC="dd:FLP/DISTSUBTIMEFRAME/0;calclus:TOF/INFOCALCLUS;cosmics:TOF/INFOCOSMICS;trkcos:TOF/INFOTRACKCOS;trksiz:TOF/INFOTRACKSIZE" -o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --dataspec "${PROXY_INSPEC}" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tof-reco-workflow --input-type raw --output-type clusters \ ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" \ diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index 5e5c3ce79..f400ca4a5 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -18,7 +18,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index 13fe859e3..cf7c83473 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -28,7 +28,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ diff --git a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh index 557b25c42..13526e29c 100755 --- a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh +++ b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh @@ -26,7 +26,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ diff --git a/DATA/testing/detectors/TPC/tpc-pattern-generator.sh b/DATA/testing/detectors/TPC/tpc-pattern-generator.sh index d6a0550ad..acc64a299 100755 --- a/DATA/testing/detectors/TPC/tpc-pattern-generator.sh +++ b/DATA/testing/detectors/TPC/tpc-pattern-generator.sh @@ -30,7 +30,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ diff --git a/DATA/testing/detectors/TRD/trd-krypton.sh b/DATA/testing/detectors/TRD/trd-krypton.sh index cfeec45d8..7c643f9b9 100755 --- a/DATA/testing/detectors/TRD/trd-krypton.sh +++ b/DATA/testing/detectors/TRD/trd-krypton.sh @@ -33,7 +33,7 @@ CTF_CONFIG+=" --require-free-disk 53687091200 --wait-for-free-disk $CTF_FREE_DIS # Start with an empty workflow WORKFLOW= -add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 add_W o2-trd-datareader "--disable-root-output --pipeline trd-datareader:$TRD_N_READERS" add_W o2-trd-kr-clusterer "--disable-root-input --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --autosave-interval 105000 --pipeline trd-kr-clusterer:8" if workflow_has_parameter QC && has_detector_qc TRD; then diff --git a/DATA/testing/detectors/TRD/trd-noise.sh b/DATA/testing/detectors/TRD/trd-noise.sh index 4b84b1ef6..cb18ed703 100755 --- a/DATA/testing/detectors/TRD/trd-noise.sh +++ b/DATA/testing/detectors/TRD/trd-noise.sh @@ -22,7 +22,7 @@ fi # Start with an empty workflow WORKFLOW= -add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 add_W o2-trd-datareader "--disable-root-output --every-nth-tf $TRD_REJECTION_FACTOR --pipeline trd-datareader:$TRD_N_READERS" add_W o2-calibration-trd-workflow "--noise --calib-dds-collection-index 0" add_W o2-calibration-ccdb-populator-workflow "--ccdb-path $TRD_CCDB_PATH" diff --git a/DATA/testing/examples/example-calib-processing.sh b/DATA/testing/examples/example-calib-processing.sh index e52adaf78..18ae83e87 100755 --- a/DATA/testing/examples/example-calib-processing.sh +++ b/DATA/testing/examples/example-calib-processing.sh @@ -9,7 +9,7 @@ source common/getCommonArgs.sh PROXY_INSPEC="A:ITS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" PROXY_OUTSPEC="downstreamA:ITS/COMPCLUSTERS/0;downstreamB:ITS/PATTERNS/0;downstreamC:ITS/CLUSTERSROF/0" -WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " +WORKFLOW="o2-dpl-raw-proxy ${ARGS_ALL} --dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=0\" | " WORKFLOW+="o2-itsmft-stf-decoder-workflow ${ARGS_ALL} --configKeyValues \"$ARGS_ALL_CONFIG\" --nthreads 4 | " WORKFLOW+="o2-dpl-output-proxy ${ARGS_ALL} --dataspec \"$PROXY_OUTSPEC\" --proxy-channel-name its-noise-input-proxy --channel-config \"name=its-noise-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0\" | " WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" diff --git a/DATA/testing/examples/example-workflow.sh b/DATA/testing/examples/example-workflow.sh index 15d5a3449..deefb230a 100755 --- a/DATA/testing/examples/example-workflow.sh +++ b/DATA/testing/examples/example-workflow.sh @@ -8,7 +8,7 @@ source common/getCommonArgs.sh PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$PROXY_INSPEC" \ diff --git a/DATA/testing/private/afurs/fdd-ft0-fv0-ctf.sh b/DATA/testing/private/afurs/fdd-ft0-fv0-ctf.sh index e8de7fd1d..b63fe32a1 100755 --- a/DATA/testing/private/afurs/fdd-ft0-fv0-ctf.sh +++ b/DATA/testing/private/afurs/fdd-ft0-fv0-ctf.sh @@ -24,7 +24,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="digfdd:FDD/DIGITSBC/0;chanfdd:FDD/DIGITSCH/0;digft0:FT0/DIGITSBC/0;chanft0:FT0/DIGITSCH/0;digfv0:FV0/DIGITSBC/0;chanfv0:FV0/DIGITSCH/0;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fdd-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ft0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ diff --git a/DATA/testing/private/afurs/fdd-ft0-fv0-digits-qc-ctf.sh b/DATA/testing/private/afurs/fdd-ft0-fv0-digits-qc-ctf.sh index f14bc2db0..9dd26457d 100755 --- a/DATA/testing/private/afurs/fdd-ft0-fv0-digits-qc-ctf.sh +++ b/DATA/testing/private/afurs/fdd-ft0-fv0-digits-qc-ctf.sh @@ -24,7 +24,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FT0/RAWDATA;y:FV0/RAWDATA;z:FDD/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fdd-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fdd-datareader-dpl:$NTHREADS \ | o2-ft0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline ft0-datareader-dpl:$NTHREADS \ | o2-fv0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fv0-datareader-dpl:$NTHREADS \ diff --git a/DATA/testing/private/afurs/ft0-fv0-ctf.sh b/DATA/testing/private/afurs/ft0-fv0-ctf.sh index f875f48f8..9430812f7 100755 --- a/DATA/testing/private/afurs/ft0-fv0-ctf.sh +++ b/DATA/testing/private/afurs/ft0-fv0-ctf.sh @@ -24,7 +24,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="digft0:FT0/DIGITSBC/0;chanft0:FT0/DIGITSCH/0;digfv0:FV0/DIGITSBC/0;chanfv0:FV0/DIGITSCH/0;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-ft0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ | o2-ctf-writer-workflow ${ARGS_ALL} ${ARGS_CTF} --configKeyValues "$ARGS_ALL_CONFIG;" --onlyDet FT0,FV0 --output-dir $CTF_DIR --ctf-dict-dir $FILEWORKDIR --output-type ctf \ diff --git a/DATA/testing/private/afurs/ft0-fv0-digits-qc-ctf.sh b/DATA/testing/private/afurs/ft0-fv0-digits-qc-ctf.sh index 0474c05e4..e22e6a622 100755 --- a/DATA/testing/private/afurs/ft0-fv0-digits-qc-ctf.sh +++ b/DATA/testing/private/afurs/ft0-fv0-digits-qc-ctf.sh @@ -24,7 +24,7 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FT0/RAWDATA;y:FV0/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" -o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ +o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fv0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fv0-datareader-dpl:$NTHREADS \ | o2-ft0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline ft0-datareader-dpl:$NTHREADS \ | o2-qc ${ARGS_ALL} --local --host epn --config json://${MYDIR}/ft0-fv0-digits-ds.json \ diff --git a/DATA/testing/private/mfasel/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh b/DATA/testing/private/mfasel/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh index e9821147f..ca917a4ca 100755 --- a/DATA/testing/private/mfasel/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh +++ b/DATA/testing/private/mfasel/runEMCRawToDigitsRecoPilelineQCAlllocalCTF.sh @@ -25,7 +25,7 @@ CTF_MINSIZE="500000000" CTF_MAX_PER_FILE=10000 o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ --severity $SEVERITY_RAWPROXY \ --infologger-severity $INFOLOGGER_SEVERITY_RAWPROXY \ diff --git a/DATA/testing/private/zampolli/zampolli-workflow.sh b/DATA/testing/private/zampolli/zampolli-workflow.sh index fe633d2e5..3406cb882 100755 --- a/DATA/testing/private/zampolli/zampolli-workflow.sh +++ b/DATA/testing/private/zampolli/zampolli-workflow.sh @@ -11,7 +11,7 @@ ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR;Na PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" \ + --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ --input-spec "$PROXY_INSPEC" \ From b272c77c615e8b164dd4b47806e5725649c2e714 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 10 Nov 2023 15:52:08 +0100 Subject: [PATCH 0183/1239] Update documentation --- DATA/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DATA/README.md b/DATA/README.md index 6b3bffbb7..f6a10ef84 100644 --- a/DATA/README.md +++ b/DATA/README.md @@ -43,6 +43,7 @@ Another abstraction layer above the *workflows* are **topology descriptions**. T ``` - Workflows should aggregate their workflow parts in the `$WORKFLOW` variable, and use `add_W` to add a workflow, and `add_QC_from_consul` to add a QC workflow with a JSON file from consul. - Calibration workflows must not pollute the production CCDB in the following case (`if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then` (bash script)), in this case please e.g. upload to `ccdb-test.cern.ch`. +- If a workflow uses an `o2-dpl-raw-proxy` to receive data from DataDistribution, that device must have the `--inject-missing-data` comand line option. # Configuring and selecting workflow in AliECS: There are 3 ways foreseen to configure the *full topology* in AliECS: (currently only the manual XML option exists) @@ -91,6 +92,7 @@ commit=xxxx|path=xxxx file=topologies.desc topology=demo-full-topology parameter Calibration workflows can be different when they use an aggregator. In that case, there is processing running on each EPN, and the output is sent to an aggregator node. Communication happens via the `o2-dpl-raw-proxy` and the `o2-dpl-output-proxy`. To set up such a workflow, a couple of points must be followed: - There are 2 different shell scripts, one for the "reco" part running on each EPN, and one for the calibration aggregator "calib" part on the calibration node. There may be more than one aggregator in the topology, in that case it is one *reco* script and multiple *calib* scripts. +- The o2-dpl-raw-proxy on a calib aggrgator workflow MUST NOT use the `--inject-missing-data` command line argument! (This is only for input proxies receiving data from DataDistribution!) - The *reco* script must contain an `o2-dpl-output-proxy` to send the output and each calib script must contain an `o2-dpl-raw-proxy` for the input. - Each of the input "raw" proxies must be assigned a unique name via the `--proxy-name [NAME]` option. Note that the *reco* script also contains an input raw proxy, with the default name `readout-proxy`. - The channel-name of each input proxy must match the proxy name. The *calib* input proxies' channels must use `method=bind`. The output proxies must use `method=connect` and the channel name must match the name of the input proxy they are connecting to. From fb2f339ec92de99ca73767dc578d81eeb6468de1 Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 13 Nov 2023 09:19:25 +0100 Subject: [PATCH 0184/1239] Remove manual material budget downloader This is no longer needed and simplifies our processing. Tasks read this directly from CCDB now. --- MC/bin/o2dpg_sim_workflow.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 72f9d1f12..954d41029 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -495,9 +495,6 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # We download some binary files, necessary for processing # Eventually, these files/objects should be queried directly from within these tasks? -MATBUD_DOWNLOADER_TASK = createTask(name='matbuddownloader', cpu='0') -MATBUD_DOWNLOADER_TASK['cmd'] = '[ -f matbud.root ] || ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p GLO/Param/MatLUT -o matbud.root --no-preserve-path --timestamp ' + str(args.timestamp) + ' --created-not-after ' + str(args.condition_not_after) -workflow['stages'].append(MATBUD_DOWNLOADER_TASK) # We download trivial TPC space charge corrections to be applied during # reco. This is necessary to have consistency (decalibration and calibration) between digitization and reconstruction ... until digitization can @@ -954,7 +951,7 @@ def getDigiTaskName(det): havePbPb = (COLTYPE == 'PbPb' or (doembedding and COLTYPEBKG == "PbPb")) ITSMemEstimate = 12000 if havePbPb else 2000 # PbPb has much large mem requirement for now (in worst case) - ITSRECOtask=createTask(name='itsreco_'+str(tf), needs=[getDigiTaskName("ITS"), MATBUD_DOWNLOADER_TASK['name']], + ITSRECOtask=createTask(name='itsreco_'+str(tf), needs=[getDigiTaskName("ITS")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem=str(ITSMemEstimate)) ITSRECOtask['cmd'] = '${O2_ROOT}/bin/o2-its-reco-workflow --trackerCA --tracking-mode async ' + getDPL_global_options(bigshm=havePbPb) \ + putConfigValuesNew(["ITSVertexerParam", "ITSAlpideParam", @@ -977,7 +974,7 @@ def getDigiTaskName(det): workflow['stages'].append(TRDTRACKINGtask) # FIXME This is so far a workaround to avoud a race condition for trdcalibratedtracklets.root - TRDTRACKINGtask2 = createTask(name='trdreco2_'+str(tf), needs=[TRDTRACKINGtask['name'],MATBUD_DOWNLOADER_TASK['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem='2000') + TRDTRACKINGtask2 = createTask(name='trdreco2_'+str(tf), needs=[TRDTRACKINGtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem='2000') TRDTRACKINGtask2['cmd'] = '${O2_ROOT}/bin/o2-trd-global-tracking ' + getDPL_global_options(bigshm=True) + ('',' --disable-mc')[args.no_mc_labels] \ + putConfigValuesNew(['ITSClustererParam', 'ITSCATrackerParam', From 2324c319f11aa2ac62b66a68cf17376ef34d56ed Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 13 Nov 2023 15:50:32 +0100 Subject: [PATCH 0185/1239] New pipeline runner production mode New production mode for the O2DPG MC workflow executor. In this mode, special end of task functions are run that permit * intermediate file cleanup * moving log files into a common archive etc. For now this is targetting to reduce a problem with too many files on HPC supercomputers. GRID jobs should enable this mode via `--production-mode` --- MC/bin/o2_dpg_workflow_runner.py | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index ecb6975d3..766cfb9e4 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -14,6 +14,7 @@ import sys import traceback import platform +import tarfile try: from graphviz import Digraph havegraphviz=True @@ -60,6 +61,8 @@ parser.add_argument('--no-rootinit-speedup', help=argparse.SUPPRESS, action='store_true') # disable init of ROOT environment vars to speedup init/startup parser.add_argument('--action-logfile', help='Logfilename for action logs. If none given, pipeline_action_#PID.log will be used') parser.add_argument('--metric-logfile', help='Logfilename for metric logs. If none given, pipeline_metric_#PID.log will be used') +parser.add_argument('--production-mode', action='store_true', help='Production mode') +# will trigger special features good for non-interactive/production processing (automatic cleanup of files etc). args = parser.parse_args() def setup_logger(name, log_file, level=logging.INFO): @@ -498,6 +501,7 @@ class WorkflowExecutor: # Constructor def __init__(self, workflowfile, args, jmax=100): self.args=args + self.is_productionmode = args.production_mode == True # os.getenv("ALIEN_PROC_ID") != None self.workflowfile = workflowfile self.workflowspec = load_json(workflowfile) self.globalenv = self.extract_global_environment(self.workflowspec) # initialize global environment settings @@ -720,7 +724,7 @@ def ok_to_submit(self, tid, backfill=False): def ok_to_skip(self, tid): done_filename = self.get_done_filename(tid) if os.path.exists(done_filename) and os.path.isfile(done_filename): - return True + return True return False def book_resources(self, tid, backfill = False): @@ -1168,6 +1172,29 @@ def produce_script(self, filename): outF.writelines(lines) outF.close() + def production_endoftask_hook(self, tid): + # Executes a hook at end of a successful task, meant to be used in GRID productions. + # For the moment, archiving away log files, done + time files from jobutils. + # TODO: In future this may be much more generic tasks such as dynamic cleanup of intermediate + # files (when they are no longer needed). + # TODO: Care must be taken with the continue feature as `_done` files are stored elsewhere now + actionlogger.info("Cleaning up log files for task " + str(tid)) + logf = self.get_logfile(tid) + donef = self.get_done_filename(tid) + timef = logf + "_time" + + # add to tar file archive + tf = tarfile.open(name="pipeline_log_archive.log.tar", mode='a') + if tf != None: + tf.add(logf) + tf.add(donef) + tf.add(timef) + tf.close() + + # remove original file + os.remove(logf) + os.remove(donef) + os.remove(timef) # print error message when no progress can be made def noprogress_errormsg(self): @@ -1299,6 +1326,12 @@ def speedup_ROOT_Init(): actionlogger.debug("finished now :" + str(finished_from_started)) finishedtasks = finishedtasks + finished + if self.is_productionmode: + # we can do some generic cleanup of finished tasks in non-interactive/GRID mode + # TODO: this can run asynchronously + for _t in finished_from_started: + self.production_endoftask_hook(_t) + # if a task was marked "failed" and we come here (because # we use --keep-going) ... we need to take out the pid from finished if len(failing) > 0: From 94ebc44ebb388a449f9efd0910608eb8bb0d0b43 Mon Sep 17 00:00:00 2001 From: shahoian Date: Mon, 13 Nov 2023 01:26:11 +0100 Subject: [PATCH 0186/1239] Change --require-tpc-lumi option to --lumi-type Following PR#12205 and PR#12237 instead of boolean switch --require-tpc-lumi one should use --lumi-type option with the meaning: 0 : no dynamic scaling 1 : require CTP lumi for TPC correction scaling 2 : require TPC scalers from CCDB for TPC correction scaling --lumi-type 1 corresponds to previous ALIEN_JDL_INSTIRFORTPC=CTP --lumi-type 2 corresponds to previous ALIEN_JDL_INSTIRFORTPC=IDCCCDB --- DATA/common/setenv.sh | 2 +- DATA/production/configurations/asyncReco/setenv_extra.sh | 9 ++++++--- UTILS/parse-async-WorkflowConfig.py | 5 +++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 2f2ca7491..06b54b207 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -97,7 +97,7 @@ if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing -if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --require-ctp-lumi, --corrmap-lumi-mean , --corrmap-lumi-inst , --ctp-lumi-factor , --ctp-lumi-source +if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --lumi-type 0 or 1 or 2, --corrmap-lumi-mean , --corrmap-lumi-inst , --ctp-lumi-factor , --ctp-lumi-source if [[ $EPNSYNCMODE == 0 ]]; then if [[ -z "${SHMSIZE:-}" ]]; then export SHMSIZE=$(( 8 << 30 )); fi # Size of shared memory for messages if [[ -z "${NGPUS:-}" ]]; then export NGPUS=1; fi # Number of GPUs to use, data distributed round-robin diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 8cdb8259f..4eb273b10 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -320,7 +320,10 @@ elif [[ $ALIGNLEVEL == 1 ]]; then return 1 fi echo "Using CTP inst lumi stored in data" - export TPC_CORR_SCALING+=" --require-ctp-lumi " + export TPC_CORR_SCALING+=" --lumi-type 1 " + elif [[ $INST_IR_FOR_TPC == "IDCCCDB" ]]; then + echo "TPC correction with IDC from CCDB will ne used" + export TPC_CORR_SCALING+=" --lumi-type 2 " else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" return 1 @@ -332,11 +335,11 @@ elif [[ $ALIGNLEVEL == 1 ]]; then if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]]; then unset TPC_CORR_SCALING - export TPC_CORR_SCALING=" --ctp-lumi-factor 2.414 --require-ctp-lumi" + export TPC_CORR_SCALING=" --ctp-lumi-factor 2.414 --lumi-type 1" if [[ $SCALE_WITH_ZDC == 0 ]]; then # scaling with FT0 if [[ $SCALE_WITH_FT0 == 1 ]]; then - export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --require-ctp-lumi " + export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --lumi-type 1 " else echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." return 1 diff --git a/UTILS/parse-async-WorkflowConfig.py b/UTILS/parse-async-WorkflowConfig.py index f5f8bb2ce..45a4c63df 100755 --- a/UTILS/parse-async-WorkflowConfig.py +++ b/UTILS/parse-async-WorkflowConfig.py @@ -193,8 +193,9 @@ def parse_important_DPL_args(cmds, flat_config): s2 = extract_args(tokens, '--corrmap-lumi-mean') if s2: corrstring += ' --corrmap-lumi-mean ' + s2 - if '--require-ctp-lumi' in tokens: - corrstring += ' --require-ctp-lumi ' + s3 = extract_args(tokens, '--lumi-type') + if s3: + corrstring += ' --lumi-type ' + s3 # these are some options applied in multiple places (so save them flatly under tpc-corr-scaling) flat_config['tpc-corr-scaling'] = corrstring From 0fcce4d2baad188dc43cd42578196f6d21b09b01 Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Mon, 13 Nov 2023 17:57:08 +0100 Subject: [PATCH 0187/1239] ITS tracker: release tracklets/cells selection for Pb-Pb tests --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 4eb273b10..016213ef4 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -372,7 +372,7 @@ export ITSEXTRAERR="ITSCATrackerParam.sysErrY2[0]=$ERRIB;ITSCATrackerParam.sysEr # ad-hoc options for ITS reco workflow EXTRA_ITSRECO_CONFIG= if [[ $BEAMTYPE == "PbPb" ]]; then - EXTRA_ITSRECO_CONFIG="ITSCATrackerParam.trackletsPerClusterLimit=5.;ITSCATrackerParam.cellsPerClusterLimit=5.;ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" + EXTRA_ITSRECO_CONFIG="ITSCATrackerParam.trackletsPerClusterLimit=15.;ITSCATrackerParam.cellsPerClusterLimit=35.;ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" elif [[ $BEAMTYPE == "pp" ]]; then EXTRA_ITSRECO_CONFIG="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" fi From f82591a1212110950bd5902ee6dbad340c6eb1f5 Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 14 Nov 2023 16:05:50 +0100 Subject: [PATCH 0188/1239] Revert "dpgsim with compatibility with old O2 (#1291)" This reverts commit 36ab46dcbec329056a59e468bb77f36d606b1667 because it seems to break GRID processing (indefinite hangs). --- MC/bin/o2dpg_sim_workflow.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 954d41029..0fa1067aa 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1309,10 +1309,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] # strangeness tracking is now called from the secondary vertexer if not args.with_strangeness_tracking: - from subprocess import run - data = run(SVFINDERtask['cmd'].split(" ")[0] + " --help", capture_output=True, shell=True, text=True) - if "disable-strangeness-tracker" in data.stdout: - SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' + SVFINDERtask['cmd'] += ' --disable-strangeness-tracker' # if enabled, it may require MC labels else: SVFINDERtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] From 0a624954f3995cf8edc44acf08d7f96ba42d7db1 Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 14 Nov 2023 16:02:08 +0100 Subject: [PATCH 0189/1239] grid_submit improvements ability to specify ErrorOut ability to specify singularity images --- GRID/utils/grid_submit.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/GRID/utils/grid_submit.sh b/GRID/utils/grid_submit.sh index e51c52b0d..7913f83ad 100755 --- a/GRID/utils/grid_submit.sh +++ b/GRID/utils/grid_submit.sh @@ -287,7 +287,7 @@ if [[ "${IS_ALIEN_JOB_SUBMITTER}" ]]; then # read preamble from job file which is used whenever command line not given - # a) OutputSpec + # -) OutputSpec [[ ! ${OUTPUTSPEC} ]] && OUTPUTSPEC=$(grep "^#JDL_OUTPUT=" ${SCRIPT} | sed 's/#JDL_OUTPUT=//') echo "Found OutputSpec to be ${OUTPUTSPEC}" if [ ! ${OUTPUTSPEC} ]; then @@ -298,7 +298,17 @@ if [[ "${IS_ALIEN_JOB_SUBMITTER}" ]]; then # check if this is a list and if all parts are properly quoted OUTPUTSPEC=$(sanitize_tokens_with_quotes ${OUTPUTSPEC}) fi - # b) PackageSpec + # -) ErrorOutputSpec + [[ ! ${ERROROUTPUTSPEC} ]] && ERROROUTPUTSPEC=$(grep "^#JDL_ERROROUTPUT=" ${SCRIPT} | sed 's/#JDL_ERROROUTPUT=//') + echo "Found ErrorOutputSpec to be ${ERROROUTPUTSPEC}" + if [ ${ERROROUTPUTSPEC} ]; then + # check if this is a list and if all parts are properly quoted + ERROROUTPUTSPEC=$(sanitize_tokens_with_quotes ${ERROROUTPUTSPEC}) + fi + # -) Special singularity / Apptainer image + [[ ! ${IMAGESPEC} ]] && IMAGESPEC=$(grep "^#JDL_IMAGE=" ${SCRIPT} | sed 's/#JDL_IMAGE=//') + echo "Found Container Image to be ${IMAGESPEC}" + # -) PackageSpec [[ ! ${PACKAGESPEC} ]] && PACKAGESPEC=$(grep "^#JDL_PACKAGE=" ${SCRIPT} | sed 's/#JDL_PACKAGE=//') echo "Found PackagesSpec to be ${PACKAGESPEC}" ## sanitize package spec @@ -341,6 +351,8 @@ TTL=${JOBTTL}; EOF echo "Output = {"${OUTPUTSPEC:-\"logs*.zip@disk=1\",\"AO2D.root@disk=1\"}"};" >> "${MY_JOBNAMEDATE}.jdl" # add output spec echo "Packages = {"${PACKAGESPEC}"};" >> "${MY_JOBNAMEDATE}.jdl" # add package spec + [ $ERROROUTPUTSPEC ] && echo "ErrorOutput = {"${ERROROUTPUTSPEC}"};" >> "${MY_JOBNAMEDATE}.jdl" # add error output files + [ $IMAGESPEC ] && echo "DebugTag = {\"${IMAGESPEC}\"};" >> "${MY_JOBNAMEDATE}.jdl" # use special singularity image to run job # "output_arch.zip:output/*@disk=2", # "checkpoint*.tar@disk=2" From b50f7d9a0101fac888d23984dfe6801d0ca92362 Mon Sep 17 00:00:00 2001 From: cterrevo Date: Fri, 10 Nov 2023 12:32:28 +0100 Subject: [PATCH 0190/1239] Adding conf and ini for D2H MC prod, adding LcpK0s and PythiaMode2 --- MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini | 2 +- MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini | 2 +- .../ini/GeneratorHF_D2H_ccbar_and_bbbar.ini | 2 +- .../GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini | 8 ++ .../PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini | 7 + .../ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini | 7 + .../pythia8_charmhadronic_with_decays.cfg | 107 +++++++++++++++ ...ythia8_charmhadronic_with_decays_Mode2.cfg | 129 ++++++++++++++++++ 8 files changed, 261 insertions(+), 3 deletions(-) create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini create mode 100644 MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg create mode 100644 MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini index e141807e0..5dcb3f77f 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini @@ -4,5 +4,5 @@ fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gapt funcName=GeneratorPythia8GapTriggeredBeauty(3, -1.5, 1.5) [GeneratorPythia8] -config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini index 4a1947cf1..a0aaba1cb 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini @@ -4,5 +4,5 @@ fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gapt funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) [GeneratorPythia8] -config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini index c159fdc6e..5e43bd908 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini @@ -4,5 +4,5 @@ fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gapt funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) [GeneratorPythia8] -config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_with_decays.cfg +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini new file mode 100755 index 000000000..56fafd009 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini new file mode 100755 index 000000000..69a2fa2fb --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini new file mode 100755 index 000000000..bdd0c2ab6 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg new file mode 100644 index 000000000..e9ce10d1d --- /dev/null +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg @@ -0,0 +1,107 @@ +### author: Fabrizio Grosa (fabrizio.grosa@cern.ch) +### since: January 2022 + +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +SoftQCD:inelastic on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### Force golden charm hadrons decay modes for trigger studies + +### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other +411:oneChannel = 1 0.0752 0 -321 211 211 +411:addChannel = 1 0.0104 0 -313 211 +411:addChannel = 1 0.0156 0 311 211 +411:addChannel = 1 0.00276 0 333 211 +## add Lc decays absent in PYTHIA8 decay table and set BRs from PDG for other +4122:oneChannel = 1 0.0196 100 2212 -313 +4122:addChannel = 1 0.0108 100 2224 -321 +4122:addChannel = 1 0.022 100 3124 211 +4122:addChannel = 1 0.035 0 2212 -321 211 +### add Xic+ decays absent in PYTHIA8 decay table +4232:addChannel = 1 0.2 0 2212 -313 +4232:addChannel = 1 0.2 0 2212 -321 211 +4232:addChannel = 1 0.2 0 3324 211 +4232:addChannel = 1 0.2 0 3312 211 211 +### add Xic0 decays absent in PYTHIA8 decay table +4132:addChannel = 1 0.2 0 3312 211 + +### K* -> K pi +313:onMode = off +313:onIfAll = 321 211 +### for Ds -> Phi pi+ +333:onMode = off +333:onIfAll = 321 321 +### for D0 -> rho0 pi+ k- +113:onMode = off +113:onIfAll = 211 211 +### for Lambda_c -> Delta++ K- +2224:onMode = off +2224:onIfAll = 2212 211 +### for Lambda_c -> Lambda(1520) K- +102134:onMode = off +102134:onIfAll = 2212 321 + +### switch off all decay channels +411:onMode = off +421:onMode = off +431:onMode = off +4112:onMode = off +4122:onMode = off +4232:onMode = off +4132:onMode = off +443:onMode = off +4332:onMode = off + +### D0 -> K pi +421:onIfMatch = 321 211 + +### D+/- -> K pi pi +411:onIfMatch = 321 211 211 +### D+/- -> K* pi +411:onIfMatch = 313 211 +### D+/- -> phi pi +411:onIfMatch = 333 211 + +### D_s -> Phi pi +431:onIfMatch = 333 211 + +### Lambda_c -> p K* +4122:onIfMatch = 2212 313 +### Lambda_c -> Delta K +4122:onIfMatch = 2224 321 +### Lambda_c -> Lambda(1520) pi +4122:onIfMatch = 3124 211 +### Lambda_c -> p K pi +4122:onIfMatch = 2212 321 211 + +### Lambda_c -> pK0s +4122:onIfMatch = 2212 311 + +### Xic+ -> pK*0 +4232:onIfMatch = 2212 313 +### Xic+ -> p K- pi+ +4232:onIfMatch = 2212 321 211 +### Xic+ -> Xi*0 pi+, Xi*->Xi- pi+ +4232:onIfMatch = 3324 211 +### Xic+ -> Xi- pi+ pi+ +4232:onIfMatch = 3312 211 211 + +### Xic0 -> Xi- pi+ +4132:onIfMatch = 3312 211 + +### Omega_c -> Omega pi +4332:onIfMatch = 3334 211 + +# Correct Lb decay length (wrong in PYTHIA8 decay table) +5122:tau0 = 4.41000e-01 + +# Correct OmegaC decay length (wrong in PYTHIA8 decay table) +4332:tau0 = 0.08000000000 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg new file mode 100644 index 000000000..801dcfa7c --- /dev/null +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg @@ -0,0 +1,129 @@ +### author: Fabrizio Grosa (fabrizio.grosa@cern.ch) +### since: January 2022 + +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +SoftQCD:inelastic on # all inelastic processes + +### decays +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +### switching on Pythia Mode2 +ColourReconnection:mode 1 +ColourReconnection:allowDoubleJunRem off +ColourReconnection:m0 0.3 +ColourReconnection:allowJunctions on +ColourReconnection:junctionCorrection 1.20 +ColourReconnection:timeDilationMode 2 +ColourReconnection:timeDilationPar 0.18 +StringPT:sigma 0.335 +StringZ:aLund 0.36 +StringZ:bLund 0.56 +StringFlav:probQQtoQ 0.078 +StringFlav:ProbStoUD 0.2 +StringFlav:probQQ1toQQ0join 0.0275,0.0275,0.0275,0.0275 +MultiPartonInteractions:pT0Ref 2.15 +BeamRemnants:remnantMode 1 +BeamRemnants:saturation 5 + + +### Force golden charm hadrons decay modes for trigger studies + +### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other +411:oneChannel = 1 0.0752 0 -321 211 211 +411:addChannel = 1 0.0104 0 -313 211 +411:addChannel = 1 0.0156 0 311 211 +411:addChannel = 1 0.00276 0 333 211 +## add Lc decays absent in PYTHIA8 decay table and set BRs from PDG for other +4122:oneChannel = 1 0.0196 100 2212 -313 +4122:addChannel = 1 0.0108 100 2224 -321 +4122:addChannel = 1 0.022 100 3124 211 +4122:addChannel = 1 0.035 0 2212 -321 211 +### add Xic+ decays absent in PYTHIA8 decay table +4232:addChannel = 1 0.2 0 2212 -313 +4232:addChannel = 1 0.2 0 2212 -321 211 +4232:addChannel = 1 0.2 0 3324 211 +4232:addChannel = 1 0.2 0 3312 211 211 +### add Xic0 decays absent in PYTHIA8 decay table +4132:addChannel = 1 0.2 0 3312 211 + +### K* -> K pi +313:onMode = off +313:onIfAll = 321 211 +### for Ds -> Phi pi+ +333:onMode = off +333:onIfAll = 321 321 +### for D0 -> rho0 pi+ k- +113:onMode = off +113:onIfAll = 211 211 +### for Lambda_c -> Delta++ K- +2224:onMode = off +2224:onIfAll = 2212 211 +### for Lambda_c -> Lambda(1520) K- +102134:onMode = off +102134:onIfAll = 2212 321 + +### switch off all decay channels +411:onMode = off +421:onMode = off +431:onMode = off +4112:onMode = off +4122:onMode = off +4232:onMode = off +4132:onMode = off +443:onMode = off +4332:onMode = off + +### D0 -> K pi +421:onIfMatch = 321 211 + +### D+/- -> K pi pi +411:onIfMatch = 321 211 211 +### D+/- -> K* pi +411:onIfMatch = 313 211 +### D+/- -> phi pi +411:onIfMatch = 333 211 + +### D_s -> Phi pi +431:onIfMatch = 333 211 + +### Lambda_c -> p K* +4122:onIfMatch = 2212 313 +### Lambda_c -> Delta K +4122:onIfMatch = 2224 321 +### Lambda_c -> Lambda(1520) pi +4122:onIfMatch = 3124 211 +### Lambda_c -> p K pi +4122:onIfMatch = 2212 321 211 + +### Lambda_c -> pK0s +4122:onIfMatch = 2212 311 + +### Xic+ -> pK*0 +4232:onIfMatch = 2212 313 +### Xic+ -> p K- pi+ +4232:onIfMatch = 2212 321 211 +### Xic+ -> Xi*0 pi+, Xi*->Xi- pi+ +4232:onIfMatch = 3324 211 +### Xic+ -> Xi- pi+ pi+ +4232:onIfMatch = 3312 211 211 + +### Xic0 -> Xi- pi+ +4132:onIfMatch = 3312 211 + +### Omega_c -> Omega pi +4332:onIfMatch = 3334 211 + +## Jpsi -> ee +443:onIfMatch = 11 11 + +# Correct Lb decay length (wrong in PYTHIA8 decay table) +5122:tau0 = 4.41000e-01 + +# Correct OmegaC decay length (wrong in PYTHIA8 decay table) +4332:tau0 = 0.08000000000 From 69b28ee80759993a015dad82c3f98ead84521f54 Mon Sep 17 00:00:00 2001 From: cterrevo Date: Fri, 10 Nov 2023 18:22:12 +0100 Subject: [PATCH 0191/1239] fixing Ds decays, Jpsi removal --- .../generator/pythia8_charmhadronic_with_decays.cfg | 8 ++++++-- .../pythia8_charmhadronic_with_decays_Mode2.cfg | 9 +++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg index e9ce10d1d..9228bba27 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg @@ -13,7 +13,7 @@ SoftQCD:inelastic on # all inelastic processes ParticleDecays:limitTau0 on ParticleDecays:tau0Max 10. -### Force golden charm hadrons decay modes for trigger studies +### Force golden charm hadrons decay modes ### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other 411:oneChannel = 1 0.0752 0 -321 211 211 @@ -25,6 +25,8 @@ ParticleDecays:tau0Max 10. 4122:addChannel = 1 0.0108 100 2224 -321 4122:addChannel = 1 0.022 100 3124 211 4122:addChannel = 1 0.035 0 2212 -321 211 +4122:addChannel = 1 0.0159 0 2212 311 +4122:addChannel = 1 0.0130 0 3122 211 ### add Xic+ decays absent in PYTHIA8 decay table 4232:addChannel = 1 0.2 0 2212 -313 4232:addChannel = 1 0.2 0 2212 -321 211 @@ -70,6 +72,8 @@ ParticleDecays:tau0Max 10. ### D+/- -> phi pi 411:onIfMatch = 333 211 +### D_s -> K K* +431:onIfMatch = 321 313 ### D_s -> Phi pi 431:onIfMatch = 333 211 @@ -78,7 +82,7 @@ ParticleDecays:tau0Max 10. ### Lambda_c -> Delta K 4122:onIfMatch = 2224 321 ### Lambda_c -> Lambda(1520) pi -4122:onIfMatch = 3124 211 +4122:onIfMatch = 102134 211 ### Lambda_c -> p K pi 4122:onIfMatch = 2212 321 211 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg index 801dcfa7c..b300ab5f1 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg @@ -44,6 +44,8 @@ BeamRemnants:saturation 5 4122:addChannel = 1 0.0108 100 2224 -321 4122:addChannel = 1 0.022 100 3124 211 4122:addChannel = 1 0.035 0 2212 -321 211 +4122:addChannel = 1 0.0159 0 2212 311 +4122:addChannel = 1 0.0130 0 3122 211 ### add Xic+ decays absent in PYTHIA8 decay table 4232:addChannel = 1 0.2 0 2212 -313 4232:addChannel = 1 0.2 0 2212 -321 211 @@ -89,6 +91,8 @@ BeamRemnants:saturation 5 ### D+/- -> phi pi 411:onIfMatch = 333 211 +### D_s -> K K* +431:onIfMatch = 321 313 ### D_s -> Phi pi 431:onIfMatch = 333 211 @@ -97,7 +101,7 @@ BeamRemnants:saturation 5 ### Lambda_c -> Delta K 4122:onIfMatch = 2224 321 ### Lambda_c -> Lambda(1520) pi -4122:onIfMatch = 3124 211 +4122:onIfMatch = 102134 211 ### Lambda_c -> p K pi 4122:onIfMatch = 2212 321 211 @@ -119,9 +123,6 @@ BeamRemnants:saturation 5 ### Omega_c -> Omega pi 4332:onIfMatch = 3334 211 -## Jpsi -> ee -443:onIfMatch = 11 11 - # Correct Lb decay length (wrong in PYTHIA8 decay table) 5122:tau0 = 4.41000e-01 From cc728940edff943209c7b52240ff0d42d3520b91 Mon Sep 17 00:00:00 2001 From: cterrevo Date: Mon, 13 Nov 2023 11:47:00 +0100 Subject: [PATCH 0192/1239] removal of duplicated ini, adding test macro --- .../ini/GeneratorHF_D2H_ccbar_and_bbbar.ini | 2 +- .../GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini | 2 +- .../PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini | 7 - .../ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini | 7 - .../tests/GeneratorHF_D2H_ccbar_and_bbbar.C | 2 +- .../GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C | 129 ++++++++++++++++++ 6 files changed, 132 insertions(+), 17 deletions(-) delete mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini delete mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini index 5e43bd908..5ae6614b6 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(3, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini index 56fafd009..1a9c4bb19 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(2, -1.5, 1.5) +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(3, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini deleted file mode 100755 index 69a2fa2fb..000000000 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3.ini +++ /dev/null @@ -1,7 +0,0 @@ -### The external generator derives from GeneratorPythia8. -[GeneratorExternal] -fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) - -[GeneratorPythia8] -config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini deleted file mode 100755 index bdd0c2ab6..000000000 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap3_Mode2.ini +++ /dev/null @@ -1,7 +0,0 @@ -### The external generator derives from GeneratorPythia8. -[GeneratorExternal] -fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) - -[GeneratorPythia8] -config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C index 68dd44347..a0652567a 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C @@ -3,7 +3,7 @@ int External() { int checkPdgQuarkOne{4}; int checkPdgQuarkTwo{5}; - float ratioTrigger = 1./2; // one event triggered out of 2 + float ratioTrigger = 1./3; // one event triggered out of 2 std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C new file mode 100644 index 000000000..a0652567a --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C @@ -0,0 +1,129 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./3; // one event triggered out of 2 + + std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}}}, // Ds+ + {443, {{-11, 11}}}, // Jpsi + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} From 6057ba4f808af179502c735d87d021f5bfa907bc Mon Sep 17 00:00:00 2001 From: cterrevo Date: Tue, 14 Nov 2023 17:54:12 +0100 Subject: [PATCH 0193/1239] fixing decays in config and test macros --- MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C | 10 +++++----- MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C | 7 +++---- .../PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C | 9 ++++----- .../ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C | 9 ++++----- .../generator/pythia8_charmhadronic_with_decays.cfg | 4 ++-- .../pythia8_charmhadronic_with_decays_Mode2.cfg | 4 +--- 6 files changed, 19 insertions(+), 24 deletions(-) diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C index 4aedceb78..66af901cc 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C @@ -1,16 +1,16 @@ -int External() { +int External() +{ std::string path{"o2sim_Kine.root"}; int checkPdgQuark{5}; float ratioTrigger = 1./3; // one event triggered out of 3 - std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ {421, {{-321, 211}, {-321, 111, 211}}}, // D0 - {431, {{211, 333}}}, // Ds+ - {443, {{-11, 11}}}, // Jpsi - {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ {4132, {{211, 3312}}}, // Xic0 {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ {4332, {{211, 3334}}} // Omegac+ diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C index 5013f4fd3..3893280f7 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C @@ -4,13 +4,12 @@ int External() { int checkPdgQuark{4}; float ratioTrigger = 1./3; // one event triggered out of 3 - std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ {421, {{-321, 211}, {-321, 111, 211}}}, // D0 - {431, {{211, 333}}}, // Ds+ - {443, {{-11, 11}}}, // Jpsi - {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ {4132, {{211, 3312}}}, // Xic0 {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ {4332, {{211, 3334}}} // Omegac+ diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C index a0652567a..79321e058 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C @@ -3,15 +3,14 @@ int External() { int checkPdgQuarkOne{4}; int checkPdgQuarkTwo{5}; - float ratioTrigger = 1./3; // one event triggered out of 2 + float ratioTrigger = 1./3; // one event triggered out of 3 - std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ {421, {{-321, 211}, {-321, 111, 211}}}, // D0 - {431, {{211, 333}}}, // Ds+ - {443, {{-11, 11}}}, // Jpsi - {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ {4132, {{211, 3312}}}, // Xic0 {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ {4332, {{211, 3334}}} // Omegac+ diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C index a0652567a..79321e058 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C @@ -3,15 +3,14 @@ int External() { int checkPdgQuarkOne{4}; int checkPdgQuarkTwo{5}; - float ratioTrigger = 1./3; // one event triggered out of 2 + float ratioTrigger = 1./3; // one event triggered out of 3 - std::vector checkPdgHadron{411, 421, 431, 443, 4122, 4132, 4232, 4332}; + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ {421, {{-321, 211}, {-321, 111, 211}}}, // D0 - {431, {{211, 333}}}, // Ds+ - {443, {{-11, 11}}}, // Jpsi - {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}}}, // Lc+ + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ {4132, {{211, 3312}}}, // Xic0 {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ {4332, {{211, 3334}}} // Omegac+ diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg index 9228bba27..d28e12d8b 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg @@ -26,7 +26,6 @@ ParticleDecays:tau0Max 10. 4122:addChannel = 1 0.022 100 3124 211 4122:addChannel = 1 0.035 0 2212 -321 211 4122:addChannel = 1 0.0159 0 2212 311 -4122:addChannel = 1 0.0130 0 3122 211 ### add Xic+ decays absent in PYTHIA8 decay table 4232:addChannel = 1 0.2 0 2212 -313 4232:addChannel = 1 0.2 0 2212 -321 211 @@ -51,6 +50,7 @@ ParticleDecays:tau0Max 10. 102134:onMode = off 102134:onIfAll = 2212 321 + ### switch off all decay channels 411:onMode = off 421:onMode = off @@ -82,7 +82,7 @@ ParticleDecays:tau0Max 10. ### Lambda_c -> Delta K 4122:onIfMatch = 2224 321 ### Lambda_c -> Lambda(1520) pi -4122:onIfMatch = 102134 211 +4122:onIfMatch = 3124 211 ### Lambda_c -> p K pi 4122:onIfMatch = 2212 321 211 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg index b300ab5f1..562c6e63f 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg @@ -45,7 +45,6 @@ BeamRemnants:saturation 5 4122:addChannel = 1 0.022 100 3124 211 4122:addChannel = 1 0.035 0 2212 -321 211 4122:addChannel = 1 0.0159 0 2212 311 -4122:addChannel = 1 0.0130 0 3122 211 ### add Xic+ decays absent in PYTHIA8 decay table 4232:addChannel = 1 0.2 0 2212 -313 4232:addChannel = 1 0.2 0 2212 -321 211 @@ -101,10 +100,9 @@ BeamRemnants:saturation 5 ### Lambda_c -> Delta K 4122:onIfMatch = 2224 321 ### Lambda_c -> Lambda(1520) pi -4122:onIfMatch = 102134 211 +4122:onIfMatch = 3124 211 ### Lambda_c -> p K pi 4122:onIfMatch = 2212 321 211 - ### Lambda_c -> pK0s 4122:onIfMatch = 2212 311 From 5096cf365fd129a3807efd1f5a3540b55bbffa9a Mon Sep 17 00:00:00 2001 From: atriolo Date: Fri, 10 Nov 2023 12:28:01 +0100 Subject: [PATCH 0194/1239] Generalizing alternate injection of quarks and hadrons --- .../generator_pythia8_gaptriggered_hf.C | 204 ++++++++++++------ .../ini/GeneratorHFTrigger_Xi_Omega_C.ini | 7 + .../pythia8_charmtriggers_xicomegac.cfg | 32 +++ ...run_pp_HFtriggers_xi_omega_c_gaptrigger.sh | 31 +++ 4 files changed, 211 insertions(+), 63 deletions(-) create mode 100644 MC/config/PWGHF/ini/GeneratorHFTrigger_Xi_Omega_C.ini create mode 100644 MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xicomegac.cfg create mode 100755 MC/run/PWGHF/run_pp_HFtriggers_xi_omega_c_gaptrigger.sh diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 0bc08dfa5..0189e53b5 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -2,37 +2,67 @@ #include "Generators/GeneratorPythia8.h" #include "Pythia8/Pythia.h" #include "TRandom.h" +#include #include #include using namespace Pythia8; -class GeneratorPythia8GapTriggeredHF : public o2::eventgen::GeneratorPythia8 { +class GeneratorPythia8GapTriggeredHF : public o2::eventgen::GeneratorPythia8 +{ public: - /// default constructor GeneratorPythia8GapTriggeredHF() = default; /// constructor - GeneratorPythia8GapTriggeredHF(int inputTriggerRatio = 5, std::vector quarkPdgList = {4}, bool doAltInjection = false) { + GeneratorPythia8GapTriggeredHF(int inputTriggerRatio = 5, std::vector quarkPdgList = {}, std::vector hadronPdgList = {}) + { mGeneratedEvents = 0; - mHadronPdg = 0; // unless differently set, we do not trigger on specific hadron species - mQuarkPdg = quarkPdgList[0]; mInverseTriggerRatio = inputTriggerRatio; mQuarkRapidityMin = -1.5; mQuarkRapidityMax = -1.5; mHadRapidityMin = -1.5; mHadRapidityMax = -1.5; - mDoNoQuarkTrigger = false; - mDoAltInjection = doAltInjection; + mDoNoQuarkTrigger = quarkPdgList.size() == 0; mQuarkPdgList = quarkPdgList; + mHadronPdgList = hadronPdgList; + if (mQuarkPdgList.size() > 1 && mHadronPdgList.size() > 1) + { + LOG(fatal)<<"GeneratorPythia8GapTriggeredHF: Only one between hadron list and quark list can have more than one element"; + } + Print(); } /// Destructor ~GeneratorPythia8GapTriggeredHF() = default; + /// Print the input + void Print() + { + std::cout << "********** GeneratorPythia8GapTriggeredHF configuration dump **********" << std::endl; + fmt::printf("* Trigger ratio: %d\n", mInverseTriggerRatio); + fmt::printf("* Quark pdg: %d\n", mQuarkPdg); + fmt::printf("* Quark rapidity: %f - %f\n", mQuarkRapidityMin, mQuarkRapidityMax); + fmt::printf("* Hadron pdg: %d\n", mHadronPdg); + fmt::printf("* Hadron rapidity: %f - %f\n", mHadRapidityMin, mHadRapidityMax); + fmt::printf("* No quark trigger: %d\n", mDoNoQuarkTrigger); + fmt::printf("* Quark pdg list: "); + for (auto pdg : mQuarkPdgList) + { + fmt::printf("%d ", pdg); + } + fmt::printf("\n"); + fmt::printf("* Hadron pdg list: "); + for (auto pdg : mHadronPdgList) + { + fmt::printf("%d ", pdg); + } + fmt::printf("\n"); + std::cout << "***********************************************************************" << std::endl; + } + bool Init() override { addSubGenerator(0, "Minimum bias"); @@ -41,8 +71,8 @@ public: return o2::eventgen::GeneratorPythia8::Init(); } - void addTriggerOnHadron(int hadPdg) { mHadronPdg = hadPdg; }; - void setQuarkTrigger (bool doNoQuarkTrigger) { mDoNoQuarkTrigger = doNoQuarkTrigger; }; + void addTriggerOnHadron(int hadPdg) { mHadronPdgList.push_back(hadPdg); }; + void setQuarkTrigger(bool doNoQuarkTrigger) { mDoNoQuarkTrigger = doNoQuarkTrigger; }; void setQuarkRapidity(float yMin, float yMax) { mQuarkRapidityMin = yMin; @@ -56,28 +86,51 @@ public: protected: //__________________________________________________________________ - bool generateEvent() override { + bool generateEvent() override + { // Simple straightforward check to alternate generators - if (mGeneratedEvents % mInverseTriggerRatio == 0) { + if (mGeneratedEvents % mInverseTriggerRatio == 0) + { + + if (mQuarkPdgList.size() > 1) + { + int indexq = (mGeneratedEvents / mInverseTriggerRatio) % mQuarkPdgList.size(); + mQuarkPdg = mQuarkPdgList[indexq]; + } + else if (mQuarkPdgList.size() == 1) + { + mQuarkPdg = mQuarkPdgList[0]; + } + + // Alternate Omega and Xi if enabled (with the same ratio) + if (mHadronPdgList.size()) + { + int indexh = (mGeneratedEvents / mInverseTriggerRatio) % mHadronPdgList.size(); + mHadronPdg = mHadronPdgList[indexh]; + } + else if (mHadronPdgList.size() == 1) + { + mHadronPdg = mHadronPdgList[0]; + } + // Generate event of interest bool genOk = false; - while (!genOk) { - if (GeneratorPythia8::generateEvent()) { + while (!genOk) + { + if (GeneratorPythia8::generateEvent()) + { genOk = selectEvent(mPythia.event); - } + } } notifySubGenerator(mQuarkPdg); - - // Alternate charm and beauty if enabled (with the same ratio) - if(mDoAltInjection) { - mQuarkPdg = (mQuarkPdg == mQuarkPdgList[0]) ? mQuarkPdgList[1] : mQuarkPdgList[0]; - } - - } else { + } + else + { // Generate minimum-bias event bool genOk = false; - while (!genOk) { + while (!genOk) + { genOk = GeneratorPythia8::generateEvent(); } notifySubGenerator(0); @@ -88,54 +141,60 @@ protected: return true; } - bool selectEvent(const Pythia8::Event& event) + bool selectEvent(const Pythia8::Event &event) { - bool isGoodAtPartonLevel; - bool isGoodAtHadronLevel; - if(mDoNoQuarkTrigger){ - isGoodAtPartonLevel = (mHadronPdg != 0) ? true : false; - isGoodAtHadronLevel = (mHadronPdg != 0) ? false : true; - } else { - isGoodAtPartonLevel = false; - isGoodAtHadronLevel = (mHadronPdg != 0) ? false : true; - } - for (auto iPart{0}; iPart < event.size(); ++iPart) { + bool isGoodAtPartonLevel{mQuarkPdgList.size() == 0}; + bool isGoodAtHadronLevel{mHadronPdgList.size() == 0}; + + for (auto iPart{0}; iPart < event.size(); ++iPart) + { // search for Q-Qbar mother with at least one Q in rapidity window - if (!isGoodAtPartonLevel) { + if (!isGoodAtPartonLevel) + { auto daughterList = event[iPart].daughterList(); bool hasQ = false, hasQbar = false, atSelectedY = false; - for (auto iDau : daughterList) { - if (event[iDau].id() == mQuarkPdg) { + for (auto iDau : daughterList) + { + if (event[iDau].id() == mQuarkPdg) + { hasQ = true; - if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) { + if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) + { atSelectedY = true; } } - if (event[iDau].id() == -mQuarkPdg) { + if (event[iDau].id() == -mQuarkPdg) + { hasQbar = true; - if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) { + if ((event[iDau].y() > mQuarkRapidityMin) && (event[iDau].y() < mQuarkRapidityMax)) + { atSelectedY = true; } - } + } } - if (hasQ && hasQbar && atSelectedY) { + if (hasQ && hasQbar && atSelectedY) + { isGoodAtPartonLevel = true; } } // search for hadron in rapidity window - if (!isGoodAtHadronLevel) { + if (!isGoodAtHadronLevel) + { int id = std::abs(event[iPart].id()); float rap = event[iPart].y(); - if (id == mHadronPdg && rap > mHadRapidityMin && rap < mHadRapidityMax) { + if (id == mHadronPdg && rap > mHadRapidityMin && rap < mHadRapidityMax) + { isGoodAtHadronLevel = true; } } // we send the trigger - if (isGoodAtPartonLevel && isGoodAtHadronLevel) { + if (isGoodAtPartonLevel && isGoodAtHadronLevel) + { + LOG(debug)<<"EVENT SELECTED: Found particle "< mQuarkPdgList; + std::vector mQuarkPdgList = {}; + + // Control alternate injection of Omega and Xi + std::vector mHadronPdgList = {}; }; // Predefined generators: // Charm-enriched -FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4}, false); +FairGenerator *GeneratorPythia8GapTriggeredCharm(int inputTriggerRatio, float yQuarkMin = -1.5, float yQuarkMax = 1.5, float yHadronMin = -1.5, float yHadronMax = 1.5, std::vector hadronPdgList = {}) +{ + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4}, hadronPdgList); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); - if(pdgCodeCharmHadron != 0) { - myGen->setQuarkTrigger(doNoQuarkTrigger); - myGen->addTriggerOnHadron(pdgCodeCharmHadron); + if (hadronPdgList.size() != 0) + { myGen->setHadronRapidity(yHadronMin, yHadronMax); } return myGen; } // Beauty-enriched -FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{5}, false); +FairGenerator *GeneratorPythia8GapTriggeredBeauty(int inputTriggerRatio, float yQuarkMin = -1.5, float yQuarkMax = 1.5, float yHadronMin = -1.5, float yHadronMax = 1.5, std::vector hadronPdgList = {}) +{ + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{5}, hadronPdgList); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); - if(pdgCodeCharmHadron != 0) { - myGen->setQuarkTrigger(doNoQuarkTrigger); - myGen->addTriggerOnHadron(pdgCodeCharmHadron); + if (hadronPdgList.size() != 0) + { myGen->setHadronRapidity(yHadronMin, yHadronMax); } return myGen; } // Charm and beauty enriched (with same ratio) -FairGenerator *GeneratorPythia8GapTriggeredCharmAndBeauty(int inputTriggerRatio, float yQuarkMin=-1.5, float yQuarkMax=1.5, float yHadronMin=-1.5, float yHadronMax=1.5, int pdgCodeCharmHadron=0, bool doNoQuarkTrigger=false) { - auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4, 5}, true); +FairGenerator *GeneratorPythia8GapTriggeredCharmAndBeauty(int inputTriggerRatio, float yQuarkMin = -1.5, float yQuarkMax = 1.5, float yHadronMin = -1.5, float yHadronMax = 1.5, std::vector hadronPdgList = {}) +{ + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, std::vector{4, 5}, hadronPdgList); auto seed = (gRandom->TRandom::GetSeed() % 900000000); myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); - if(pdgCodeCharmHadron != 0) { - myGen->setQuarkTrigger(doNoQuarkTrigger); - myGen->addTriggerOnHadron(pdgCodeCharmHadron); + if (hadronPdgList.size() != 0) + { myGen->setHadronRapidity(yHadronMin, yHadronMax); } return myGen; } + +FairGenerator *GeneratorPythia8GapHF(int inputTriggerRatio, float yQuarkMin = -1.5, float yQuarkMax = 1.5, float yHadronMin = -1.5, float yHadronMax = 1.5, std::vector quarkPdgList = {}, std::vector hadronPdgList = {}) +{ + if (hadronPdgList.size() == 0 && quarkPdgList.size() == 0) + { + LOG(fatal) << "GeneratorPythia8GapHF: At least one quark or hadron PDG code must be specified"; + } + auto myGen = new GeneratorPythia8GapTriggeredHF(inputTriggerRatio, quarkPdgList, hadronPdgList); + auto seed = (gRandom->TRandom::GetSeed() % 900000000); + myGen->readString("Random:setSeed on"); + myGen->readString("Random:seed " + std::to_string(seed)); + myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); + myGen->setQuarkTrigger(quarkPdgList.size()); + myGen->setHadronRapidity(yHadronMin, yHadronMax); + + return myGen; +} \ No newline at end of file diff --git a/MC/config/PWGHF/ini/GeneratorHFTrigger_Xi_Omega_C.ini b/MC/config/PWGHF/ini/GeneratorHFTrigger_Xi_Omega_C.ini new file mode 100644 index 000000000..a1914e6e4 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHFTrigger_Xi_Omega_C.ini @@ -0,0 +1,7 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapHF(5,-1.5,1.5,-1.5,1.5,{4}, {4132, 4332}) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xicomegac.cfg \ No newline at end of file diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xicomegac.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xicomegac.cfg new file mode 100644 index 000000000..7bb053f28 --- /dev/null +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xicomegac.cfg @@ -0,0 +1,32 @@ +### Xi_c + +### beams +Beams:idA 2212 # proton +Beams:idB 2212 # proton +Beams:eCM 13600. # GeV + +### processes +SoftQCD:inelastic on # all inelastic processes + +ParticleDecays:limitTau0 on +ParticleDecays:tau0Max 10. + +# Correct XiC decay length (mm/c) +4132:tau0 = 0.04557000000 + +### add XiC decay absent in PYTHIA8 decay table +4132:addChannel = 1 0.02 0 3312 211 + +### force the XiC to decay in the XiC -> Xi pi channel +4132:onMode = off +4132:onIfMatch = 3312 211 + +# Correct OmegaC decay length (wrong in PYTHIA8 decay table) (mm/c) +4332:tau0 = 0.08034000000 + +### add OmegaC decay absent in PYTHIA8 decay table +4332:addChannel = 1 0.02 0 3334 211 + +### force the OmegaC to decay in the Omega_c -> Omega pi channel +4332:onMode = off +4332:onIfMatch = 3334 211 \ No newline at end of file diff --git a/MC/run/PWGHF/run_pp_HFtriggers_xi_omega_c_gaptrigger.sh b/MC/run/PWGHF/run_pp_HFtriggers_xi_omega_c_gaptrigger.sh new file mode 100755 index 000000000..9ecb4d3d8 --- /dev/null +++ b/MC/run/PWGHF/run_pp_HFtriggers_xi_omega_c_gaptrigger.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +CPU_LIMIT=${CPU_LIMIT:-32} +MODULES="--skipModules ZDC" +SIMENGINE=${SIMENGINE:-TGeant4} +NSIGEVENTS=${NSIGEVENTS:-30} +NTIMEFRAMES=${NTIMEFRAMES:-1} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" +# create workflow + +#ccbar filter +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -col pp -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate 500000 -confKey "Diamond.width[2]=6." -e ${SIMENGINE} ${SEED} -mod "--skipModules ZDC" \ + -ini $O2DPG_ROOT/MC/config/PWGHF/ini/GeneratorHFTrigger_Xi_Omega_C.ini \ + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit ${CPU_LIMIT} \ No newline at end of file From ebd9af4859a9245aac15fa65bb036140034884f4 Mon Sep 17 00:00:00 2001 From: atriolo Date: Fri, 10 Nov 2023 17:12:14 +0100 Subject: [PATCH 0195/1239] Adding test script for GeneratorHFTrigger_Xi_Omega_C.ini --- .../ini/tests/GeneratorHFTrigger_Xi_Omega_C.C | 238 ++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C diff --git a/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C b/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C new file mode 100644 index 000000000..c33087f85 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C @@ -0,0 +1,238 @@ +int External() +{ + int checkPdgSignal = 4132; + int checkPdgDecayPi = 211; + int checkPdgDecayXi = 3312; + int checkPdgDecayLambda = 3122; + int checkPdgDecayP = 2212; + int checkPdgQuark = 4; + float ratioTrigger = 1./3; // one event triggered out of 3 + + std::string path{"o2sim_Kine.root"}; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecayPi << " and " << checkPdgDecayPi << "\n"; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) + { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nQuark{}; // charm quark + int nXi{}; // xi- + int nAntiXi{}; // xi+ + int nPi{}; // pi+ + int nAntiPi{}; // pi- + + int nDauPairs{}; + + int nSignalTot{}; + int nSignalPart{}; + int nSignalAntiPart{}; + + int nDecayXic{}; + int nDecayXi{}; + int nDecayLambda{}; + int nFullDecayChain{}; + + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) + { + tree->GetEntry(i); + for (auto &track : *tracks) + { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuark) { + nQuark++; + continue; + } + if (pdg == checkPdgDecayXi) + { + nXi++; + } + else if (pdg == -checkPdgDecayXi) + { + nAntiXi++; + } + else if (pdg == checkPdgDecayPi) + { + nPi++; + } + else if (pdg == -checkPdgDecayPi) + { + nAntiPi++; + } + else if (std::abs(pdg) == checkPdgSignal) + { + nSignalTot++; + + if(pdg == checkPdgSignal){ + nSignalPart++; + } else if (pdg == -checkPdgSignal) { + nSignalAntiPart++; + } + + auto child0 = o2::mcutils::MCTrackNavigator::getDaughter0(track, *tracks); + auto child1 = o2::mcutils::MCTrackNavigator::getDaughter1(track, *tracks); + if (child0 != nullptr && child1 != nullptr) + { + nDauPairs++; + + // check for parent-child relations + auto pdg0 = child0->GetPdgCode(); + auto pdg1 = child1->GetPdgCode(); + std::cout << "First and last children of parent " << pdg << " are PDG0: " << pdg0 << " PDG1: " << pdg1 << "\n"; + if ((std::abs(pdg0) == checkPdgDecayXi && std::abs(pdg1) == checkPdgDecayPi) || (std::abs(pdg1) == checkPdgDecayXi && std::abs(pdg0) == checkPdgDecayPi)) + { // 211 pi+ and 3312 xi- from MC numbering scheme + nDecayXic++; + // ------------- cascade is child0 ------------- + if (std::abs(pdg0) == checkPdgDecayXi) + { + auto childCasc0 = o2::mcutils::MCTrackNavigator::getDaughter0(*child0, *tracks); + auto childCasc1 = o2::mcutils::MCTrackNavigator::getDaughter1(*child0, *tracks); + if (childCasc0 != nullptr && childCasc1 != nullptr) + { + if ((std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayXi++; + // lambda is childCasc0 + if (std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda) + { + auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc0, *tracks); + auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc0, *tracks); + if (childLam0 != nullptr && childLam1 != nullptr) + { + if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayLambda++; + nFullDecayChain++; + } + } + } + else if (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda) + { // lambda is childCasc1 + auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc1, *tracks); + auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc1, *tracks); + if (childLam0 != nullptr && childLam1 != nullptr) + { + if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayLambda++; + nFullDecayChain++; + } + } + } + } + } + } + + else if (std::abs(pdg1) == checkPdgDecayXi) + { // ------------- cascade is child1 ------------- + auto childCasc0 = o2::mcutils::MCTrackNavigator::getDaughter0(*child1, *tracks); + auto childCasc1 = o2::mcutils::MCTrackNavigator::getDaughter1(*child1, *tracks); + if (childCasc0 != nullptr && childCasc1 != nullptr) + { + if ((std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayXi++; + // lambda is chilCasc0 + if (std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda) + { + auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc0, *tracks); + auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc0, *tracks); + if (childLam0 != nullptr && childLam1 != nullptr) + { + if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayLambda++; + nFullDecayChain++; + } + } + } + else if (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda) + { // lambda is childCasc1 + auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc1, *tracks); + auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc1, *tracks); + if (childLam0 != nullptr && childLam1 != nullptr) + { + if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) + { + nDecayLambda++; + nFullDecayChain++; + } + } + } + } + } + } + } + } + } + } + } + + + std::cout << "#events: " << nEvents << "\n" + <<"#charm quark: " << nQuark << "\n" + << "#xi: " << nXi << "\n" + << "#antixi: " << nAntiXi << "\n" + << "#pi: " << nPi << "\n" + << "#antipi: " << nAntiPi << "\n" + << "#signal tot: " << nSignalTot << "\n" + << "#signal particles: " << nSignalPart << "\n" + << "#signal anti-particles: " << nSignalAntiPart << "\n" + << "#Daughter pairs: " << nDauPairs << "\n" + << "#Correct Xic decays: " << nDecayXic << "\n" + << "#Correct Xi decays: " << nDecayXi << "\n" + << "#Correct Lambda decays: " << nDecayLambda << "\n" + << "#Correct full decay chain: " << nFullDecayChain << "\n"; + + if (nDauPairs == 0) + { + std::cerr << "Number of daughter pairs should be greater than 0.\n"; + return 1; + } + if (nSignalTot == 0) + { + std::cerr << "Number of Xic + Anti-Xic should be greater than 0.\n"; + return 1; + } + if (nXi == 0 && nAntiXi == 0) + { + std::cerr << "At least one among number of xi and number of anti-xi should be greater than 1.\n"; + return 1; + } + if (nPi == 0 && nAntiPi == 0) + { + std::cerr << "At least one among number of pi and number of anti-pi should be greater than 1.\n"; + return 1; + } + // check all the steps in the decay chain + if (nDecayXic != nDecayXi) + { + std::cerr << "The Xi decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi).\n"; + return 1; + } + if (nDecayXic != nDecayLambda) + { + std::cerr << "The Lambda decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi -> ((p pi) pi) pi).\n"; + return 1; + } + if (nDecayXic != nFullDecayChain) + { + std::cerr << "The full OmegaC decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi -> ((p pi) pi) pi).\n"; + return 1; + } + if (nQuark < 2 * nEvents * ratioTrigger) // we expect anyway more because the same quark is repeated several time, after each gluon radiation + { + std::cerr << "Number of generated (anti)quarks " << checkPdgQuark << " lower than expected\n"; + return 1; + } + + return 0; + +} From 0afed286a36ff48a35f18d2314e2be92ab70a61c Mon Sep 17 00:00:00 2001 From: atriolo Date: Fri, 10 Nov 2023 17:31:24 +0100 Subject: [PATCH 0196/1239] Modifying arguments of GeneratorPythia8GapTriggeredCharm called in already existing ini files --- MC/config/PWGDQ/ini/GeneratorJPsiHFCorr_ccbar.ini | 2 +- MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini | 2 +- MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MC/config/PWGDQ/ini/GeneratorJPsiHFCorr_ccbar.ini b/MC/config/PWGDQ/ini/GeneratorJPsiHFCorr_ccbar.ini index 6db692633..8c0f3b311 100755 --- a/MC/config/PWGDQ/ini/GeneratorJPsiHFCorr_ccbar.ini +++ b/MC/config/PWGDQ/ini/GeneratorJPsiHFCorr_ccbar.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(3, -6, 6, -4.5, -2., 443) +funcName=GeneratorPythia8GapTriggeredCharm(3, -6, 6, -4.5, -2., {443}) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGDQ/pythia8/decayer/jpsi_hf_corr.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini b/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini index 7c09045ee..d36bf8025 100644 --- a/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini +++ b/MC/config/PWGHF/ini/GeneratorHFTrigger_OmegaCToXiPi.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4332) +funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,{4332}) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_omegactoxipi.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini b/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini index d5915c399..89b12fbd9 100644 --- a/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini +++ b/MC/config/PWGHF/ini/GeneratorHFTrigger_XiCToXiPi.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,4132) +funcName=GeneratorPythia8GapTriggeredCharm(5,-1.5,1.5,-1.5,1.5,{4132}) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmtriggers_xictoxipi.cfg From 24b04f98fc2e0494ff6ada0f8f96334cfc73e87a Mon Sep 17 00:00:00 2001 From: atriolo Date: Tue, 14 Nov 2023 11:25:00 +0100 Subject: [PATCH 0197/1239] code cleanup --- .../generator_pythia8_gaptriggered_hf.C | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 0189e53b5..7db624bd3 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -25,7 +25,8 @@ public: mQuarkRapidityMax = -1.5; mHadRapidityMin = -1.5; mHadRapidityMax = -1.5; - mDoNoQuarkTrigger = quarkPdgList.size() == 0; + mQuarkPdg = 0; + mHadronPdg = 0; mQuarkPdgList = quarkPdgList; mHadronPdgList = hadronPdgList; if (mQuarkPdgList.size() > 1 && mHadronPdgList.size() > 1) @@ -41,26 +42,23 @@ public: /// Print the input void Print() { - std::cout << "********** GeneratorPythia8GapTriggeredHF configuration dump **********" << std::endl; - fmt::printf("* Trigger ratio: %d\n", mInverseTriggerRatio); - fmt::printf("* Quark pdg: %d\n", mQuarkPdg); - fmt::printf("* Quark rapidity: %f - %f\n", mQuarkRapidityMin, mQuarkRapidityMax); - fmt::printf("* Hadron pdg: %d\n", mHadronPdg); - fmt::printf("* Hadron rapidity: %f - %f\n", mHadRapidityMin, mHadRapidityMax); - fmt::printf("* No quark trigger: %d\n", mDoNoQuarkTrigger); - fmt::printf("* Quark pdg list: "); + LOG(info) << "********** GeneratorPythia8GapTriggeredHF configuration dump **********"; + LOG(info)< 1) { int indexq = (mGeneratedEvents / mInverseTriggerRatio) % mQuarkPdgList.size(); @@ -103,7 +99,7 @@ protected: mQuarkPdg = mQuarkPdgList[0]; } - // Alternate Omega and Xi if enabled (with the same ratio) + // Alternate hadrons if enabled (with the same ratio) if (mHadronPdgList.size()) { int indexh = (mGeneratedEvents / mInverseTriggerRatio) % mHadronPdgList.size(); @@ -207,22 +203,21 @@ private: Pythia8::Event mOutputEvent; // Properties of selection - int mQuarkPdg = 0; + int mQuarkPdg; float mQuarkRapidityMin; float mQuarkRapidityMax; - int mHadronPdg = 0; + int mHadronPdg; float mHadRapidityMin; float mHadRapidityMax; - bool mDoNoQuarkTrigger; // Control gap-triggering unsigned long long mGeneratedEvents; int mInverseTriggerRatio; - // Control alternate injection of charm and beauty + // Control alternate trigger on charm and beauty quarks std::vector mQuarkPdgList = {}; - // Control alternate injection of Omega and Xi + // Control alternate trigger on different hadrons std::vector mHadronPdgList = {}; }; @@ -283,7 +278,6 @@ FairGenerator *GeneratorPythia8GapHF(int inputTriggerRatio, float yQuarkMin = -1 myGen->readString("Random:setSeed on"); myGen->readString("Random:seed " + std::to_string(seed)); myGen->setQuarkRapidity(yQuarkMin, yQuarkMax); - myGen->setQuarkTrigger(quarkPdgList.size()); myGen->setHadronRapidity(yHadronMin, yHadronMax); return myGen; From 2059c95dc994a785a58d9d775a8f8779540627be Mon Sep 17 00:00:00 2001 From: atriolo Date: Tue, 14 Nov 2023 12:32:21 +0100 Subject: [PATCH 0198/1239] Adding possibility to trigger both on multiple quarks and hadrons --- .../generator_pythia8_gaptriggered_hf.C | 29 +++++++------------ .../ini/GeneratorHFTrigger_Xi_Omega_C.ini | 2 +- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 7db624bd3..559bc558e 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -29,10 +29,6 @@ public: mHadronPdg = 0; mQuarkPdgList = quarkPdgList; mHadronPdgList = hadronPdgList; - if (mQuarkPdgList.size() > 1 && mHadronPdgList.size() > 1) - { - LOG(fatal)<<"GeneratorPythia8GapTriggeredHF: Only one between hadron list and quark list can have more than one element"; - } Print(); } @@ -84,30 +80,25 @@ protected: //__________________________________________________________________ bool generateEvent() override { + // Simple straightforward check to alternate generators if (mGeneratedEvents % mInverseTriggerRatio == 0) { + int nInjectedEvents = mGeneratedEvents / mInverseTriggerRatio; // Alternate quarks if enabled (with the same ratio) - if (mQuarkPdgList.size() > 1) - { - int indexq = (mGeneratedEvents / mInverseTriggerRatio) % mQuarkPdgList.size(); - mQuarkPdg = mQuarkPdgList[indexq]; - } - else if (mQuarkPdgList.size() == 1) + if (mQuarkPdgList.size() >= 1) { - mQuarkPdg = mQuarkPdgList[0]; + int iQuark = nInjectedEvents % mQuarkPdgList.size(); + mQuarkPdg = mQuarkPdgList[iQuark]; + LOG(debug)<<"SELECTED quark: "<= 1) { - mHadronPdg = mHadronPdgList[0]; + int iHadron = (nInjectedEvents / std::max(mHadronPdgList.size(), 1ul)) % mHadronPdgList.size(); + mHadronPdg = mHadronPdgList[iHadron]; + LOG(debug)<<"SELECTED hadron: "< Date: Wed, 15 Nov 2023 11:51:19 +0100 Subject: [PATCH 0199/1239] Fix bug in test for GeneratorHFTrigger_Xi_Omega_C --- .../ini/tests/GeneratorHFTrigger_Xi_Omega_C.C | 91 +------------------ 1 file changed, 4 insertions(+), 87 deletions(-) diff --git a/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C b/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C index c33087f85..1e0a637a9 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHFTrigger_Xi_Omega_C.C @@ -3,13 +3,11 @@ int External() int checkPdgSignal = 4132; int checkPdgDecayPi = 211; int checkPdgDecayXi = 3312; - int checkPdgDecayLambda = 3122; - int checkPdgDecayP = 2212; int checkPdgQuark = 4; float ratioTrigger = 1./3; // one event triggered out of 3 std::string path{"o2sim_Kine.root"}; - std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecayPi << " and " << checkPdgDecayPi << "\n"; + std::cout << "Check for\nsignal PDG " << checkPdgSignal << "\ndecay PDG " << checkPdgDecayXi << " and " << checkPdgDecayPi << "\n"; TFile file(path.c_str(), "READ"); if (file.IsZombie()) { @@ -34,8 +32,6 @@ int External() int nSignalAntiPart{}; int nDecayXic{}; - int nDecayXi{}; - int nDecayLambda{}; int nFullDecayChain{}; auto nEvents = tree->GetEntries(); @@ -94,80 +90,14 @@ int External() { auto childCasc0 = o2::mcutils::MCTrackNavigator::getDaughter0(*child0, *tracks); auto childCasc1 = o2::mcutils::MCTrackNavigator::getDaughter1(*child0, *tracks); - if (childCasc0 != nullptr && childCasc1 != nullptr) - { - if ((std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayXi++; - // lambda is childCasc0 - if (std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda) - { - auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc0, *tracks); - auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc0, *tracks); - if (childLam0 != nullptr && childLam1 != nullptr) - { - if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayLambda++; - nFullDecayChain++; - } - } - } - else if (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda) - { // lambda is childCasc1 - auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc1, *tracks); - auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc1, *tracks); - if (childLam0 != nullptr && childLam1 != nullptr) - { - if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayLambda++; - nFullDecayChain++; - } - } - } - } - } + nFullDecayChain++; } else if (std::abs(pdg1) == checkPdgDecayXi) { // ------------- cascade is child1 ------------- auto childCasc0 = o2::mcutils::MCTrackNavigator::getDaughter0(*child1, *tracks); auto childCasc1 = o2::mcutils::MCTrackNavigator::getDaughter1(*child1, *tracks); - if (childCasc0 != nullptr && childCasc1 != nullptr) - { - if ((std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda && std::abs(childCasc0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayXi++; - // lambda is chilCasc0 - if (std::abs(childCasc0->GetPdgCode()) == checkPdgDecayLambda) - { - auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc0, *tracks); - auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc0, *tracks); - if (childLam0 != nullptr && childLam1 != nullptr) - { - if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayLambda++; - nFullDecayChain++; - } - } - } - else if (std::abs(childCasc1->GetPdgCode()) == checkPdgDecayLambda) - { // lambda is childCasc1 - auto childLam0 = o2::mcutils::MCTrackNavigator::getDaughter0(*childCasc1, *tracks); - auto childLam1 = o2::mcutils::MCTrackNavigator::getDaughter1(*childCasc1, *tracks); - if (childLam0 != nullptr && childLam1 != nullptr) - { - if ((std::abs(childLam0->GetPdgCode()) == checkPdgDecayP && std::abs(childLam1->GetPdgCode()) == checkPdgDecayPi) || (std::abs(childLam1->GetPdgCode()) == checkPdgDecayP && std::abs(childLam0->GetPdgCode()) == checkPdgDecayPi)) - { - nDecayLambda++; - nFullDecayChain++; - } - } - } - } - } + nFullDecayChain++; } } } @@ -187,8 +117,6 @@ int External() << "#signal anti-particles: " << nSignalAntiPart << "\n" << "#Daughter pairs: " << nDauPairs << "\n" << "#Correct Xic decays: " << nDecayXic << "\n" - << "#Correct Xi decays: " << nDecayXi << "\n" - << "#Correct Lambda decays: " << nDecayLambda << "\n" << "#Correct full decay chain: " << nFullDecayChain << "\n"; if (nDauPairs == 0) @@ -211,20 +139,9 @@ int External() std::cerr << "At least one among number of pi and number of anti-pi should be greater than 1.\n"; return 1; } - // check all the steps in the decay chain - if (nDecayXic != nDecayXi) - { - std::cerr << "The Xi decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi).\n"; - return 1; - } - if (nDecayXic != nDecayLambda) - { - std::cerr << "The Lambda decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi -> ((p pi) pi) pi).\n"; - return 1; - } if (nDecayXic != nFullDecayChain) { - std::cerr << "The full OmegaC decay chain is not the expected one (Xic -> Xi pi -> (Lambda pi) pi -> ((p pi) pi) pi).\n"; + std::cerr << "The full OmegaC decay chain is not the expected one (Xic -> Xi pi).\n"; return 1; } if (nQuark < 2 * nEvents * ratioTrigger) // we expect anyway more because the same quark is repeated several time, after each gluon radiation From 3a712a8274ae30941c32b39062b967a5e3c26f24 Mon Sep 17 00:00:00 2001 From: atriolo Date: Wed, 15 Nov 2023 15:48:24 +0100 Subject: [PATCH 0200/1239] Fix alternate injection for more than two hadrons --- .../external/generator/generator_pythia8_gaptriggered_hf.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C index 559bc558e..462ae589b 100644 --- a/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +++ b/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C @@ -96,7 +96,7 @@ protected: // Alternate hadrons if enabled (with the same ratio) if (mHadronPdgList.size() >= 1) { - int iHadron = (nInjectedEvents / std::max(mHadronPdgList.size(), 1ul)) % mHadronPdgList.size(); + int iHadron = (nInjectedEvents / std::max(mQuarkPdgList.size(), 1ul)) % mHadronPdgList.size(); mHadronPdg = mHadronPdgList[iHadron]; LOG(debug)<<"SELECTED hadron: "< Date: Fri, 17 Nov 2023 13:10:15 +0100 Subject: [PATCH 0201/1239] Update TRD QC json for DPL CCDB fetcher (#1332) --- DATA/production/qc-async/trd.json | 6 +----- DATA/production/qc-sync/trd.json | 9 ++------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index bf81c26b8..8af525aa9 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -29,10 +29,9 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;triggers:TRD/TRKTRGRD" + "query": "digits:TRD/DIGITS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" } }, "Tracklets": { @@ -41,7 +40,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" @@ -53,7 +51,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "phValues:TRD/PULSEHEIGHT" @@ -65,7 +62,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC" diff --git a/DATA/production/qc-sync/trd.json b/DATA/production/qc-sync/trd.json index cdeb487f7..019762db5 100644 --- a/DATA/production/qc-sync/trd.json +++ b/DATA/production/qc-sync/trd.json @@ -34,7 +34,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "rawstats:TRD/RAWSTATS" @@ -56,7 +55,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "trdall" @@ -82,7 +80,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "trdall" @@ -109,7 +106,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "phValues:TRD/PULSEHEIGHT" @@ -134,7 +130,6 @@ "moduleName": "QcTRD", "detectorName": "TRD", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC" @@ -164,12 +159,12 @@ "id": "trdall", "active": "true", "machines": [], - "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0", + "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC", "samplingConditions": [ { "condition": "random", "fraction": "0.05", - "seed": "1234" + "seed": "0" } ], "blocking": "false" From 29a214ad9a65bd9469bf7f2f3f586c394c82ef4c Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Sat, 21 Oct 2023 12:37:32 +0200 Subject: [PATCH 0202/1239] Add new configurable cuts for GLO QC --- DATA/production/qc-async/itstpc.json | 17 ++++++++++++----- .../QC/json/ITSTPCmatchedTracks_direct_MC.json | 17 ++++++++++++----- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/DATA/production/qc-async/itstpc.json b/DATA/production/qc-async/itstpc.json index 9abcd46c7..50ad83d1b 100644 --- a/DATA/production/qc-async/itstpc.json +++ b/DATA/production/qc-async/itstpc.json @@ -44,11 +44,18 @@ "taskParameters" : { "GID" : "ITS-TPC,ITS", "verbose" : "false", - "minPtCut" : "0.1f", - "etaCut" : "1.4f", - "minNTPCClustersCut" : "60", - "minDCACut" : "100.f", - "minDCACutY" : "10.f" + "minPtITSCut": "0.f", + "etaITSCut": "1e10f", + "minNITSClustersCut": "0", + "maxChi2PerClusterITS": "1e10f", + "minPtTPCCut": "0.1f", + "etaTPCCut": "1.4f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f", + "minPtCut": "0.f", + "maxPtCut": "1e10f", + "etaCut": "1e10f" }, "grpGeomRequest" : { "geomRequest": "Aligned", diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index 9f6f9eae6..d29c247c0 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -46,11 +46,18 @@ "taskParameters" : { "GID" : "ITS-TPC,ITS", "verbose" : "false", - "minPtCut" : "0.1f", - "etaCut" : "1.4f", - "minNTPCClustersCut" : "60", - "minDCACut" : "100.f", - "minDCACutY" : "10.f", + "minPtITSCut": "0.f", + "etaITSCut": "1e10f", + "minNITSClustersCut": "0", + "maxChi2PerClusterITS": "1e10f", + "minPtTPCCut": "0.1f", + "etaTPCCut": "1.4f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f", + "minPtCut": "0.f", + "maxPtCut": "1e10f", + "etaCut": "1e10f", "isMC" : "true" }, "grpGeomRequest" : { From d847f5e7e33e62544c338fe07ba072d556fb60e7 Mon Sep 17 00:00:00 2001 From: Matthias Kleiner Date: Thu, 16 Nov 2023 14:12:40 +0100 Subject: [PATCH 0203/1239] TPC timeseries: Adding option to store unbinned data --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 016213ef4..2848633be 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -542,6 +542,12 @@ fi if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-time-series-workflow" + if [[ ! -z "$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES" ]]; then + export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --enable-unbinned-root-output --sample-unbinned-tsallis --threads 1" + fi + if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then + export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${ALIEN_JDL_SAMPLINGFACTORTIMESERIES}" + fi fi # Enabling AOD From 111449e5536d9df9855e97840981c97d66c662a4 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Fri, 17 Nov 2023 18:51:43 +0100 Subject: [PATCH 0204/1239] TRD async QC json update CCDB objects (#1334) --- DATA/production/qc-async/trd.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index 8af525aa9..1b37d231e 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -42,7 +42,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" + "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" } }, "PHTrackMatch": { From 2101322eb6c01ad0da508004a786f70f48bd1bb7 Mon Sep 17 00:00:00 2001 From: shahoian Date: Fri, 17 Nov 2023 18:28:00 +0100 Subject: [PATCH 0205/1239] Adapt TPC_CORR_SCALING to PR12264 --- DATA/common/setenv.sh | 2 +- .../configurations/asyncReco/setenv_extra.sh | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 06b54b207..a7e41c6a8 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -97,7 +97,7 @@ if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing -if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --lumi-type 0 or 1 or 2, --corrmap-lumi-mean , --corrmap-lumi-inst , --ctp-lumi-factor , --ctp-lumi-source +if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1> and TPCCorrMap... configurable param if [[ $EPNSYNCMODE == 0 ]]; then if [[ -z "${SHMSIZE:-}" ]]; then export SHMSIZE=$(( 8 << 30 )); fi # Size of shared memory for messages if [[ -z "${NGPUS:-}" ]]; then export NGPUS=1; fi # Number of GPUs to use, data distributed round-robin diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 2848633be..82006fc5d 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -303,17 +303,17 @@ elif [[ $ALIGNLEVEL == 1 ]]; then # now we set the options if [[ $INST_IR_FOR_TPC -gt 0 ]]; then # externally imposed IR for scaling echo "Applying externally provided IR for scaling, $INST_IR_FOR_TPC Hz" - export TPC_CORR_SCALING+=" --corrmap-lumi-inst $INST_IR_FOR_TPC " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" elif [[ $INST_IR_FOR_TPC == 0 ]]; then # when zero, only the TPC/Calib/CorrectionMaps is applied echo "Passed valued for scaling is zero, only TPC/Calib/CorrectionMaps will be applied" - export TPC_CORR_SCALING+=" --corrmap-lumi-inst $INST_IR_FOR_TPC " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" elif [[ $INST_IR_FOR_TPC -lt 0 ]]; then # do not apply any correction echo "Passed valued for scaling is smaller than zero, no scaling will be applied" echo "NOTA BENE: In the future, this value will signal to not apply any correction at all, which is not operational yet (but please check, as it depends on O2)" - export TPC_CORR_SCALING+=" --corrmap-lumi-inst $INST_IR_FOR_TPC " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" elif [[ $INST_IR_FOR_TPC == "CTPCCDB" ]]; then # using what we have in the CCDB CTP counters, extracted at the beginning of the script echo "Using CTP CCDB which gave the mean IR of the run at the beginning of the script ($RUN_IR Hz)" - export TPC_CORR_SCALING+=" --corrmap-lumi-inst $RUN_IR " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$RUN_IR" elif [[ $INST_IR_FOR_TPC == "CTP" ]]; then if ! has_detector CTP ; then echo "TPC correction with CTP Lumi is requested but CTP is not in the WORKFLOW_DETECTORS=$WORKFLOW_DETECTORS" @@ -330,16 +330,16 @@ elif [[ $ALIGNLEVEL == 1 ]]; then fi if [[ -n $ALIEN_JDL_MEANIRFORTPC && $ALIEN_JDL_MEANIRFORTPC > 0 ]]; then # externally imposed TPC map mean IR for scaling - export TPC_CORR_SCALING+=" --corrmap-lumi-mean $ALIEN_JDL_MEANIRFORTPC " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$ALIEN_JDL_MEANIRFORTPC" fi if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]]; then unset TPC_CORR_SCALING - export TPC_CORR_SCALING=" --ctp-lumi-factor 2.414 --lumi-type 1" + export TPC_CORR_SCALING=";TPCCorrMap.lumiInstFactor=2.414 --lumi-type 1" if [[ $SCALE_WITH_ZDC == 0 ]]; then # scaling with FT0 if [[ $SCALE_WITH_FT0 == 1 ]]; then - export TPC_CORR_SCALING=" --ctp-lumi-source 1 --ctp-lumi-factor 135. --lumi-type 1 " + export TPC_CORR_SCALING=" --ctp-lumi-source 1 --lumi-type 1 TPCCorrMap.lumiInstFactor=135." else echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." return 1 From 3cf5219f6bccc7adc89281105bab3d5daa38986d Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Mon, 20 Nov 2023 09:48:37 +0100 Subject: [PATCH 0206/1239] Clean and adapt config files for EventaTrack AnalysisQC (#1331) - clean all obsolete Configurables - remove most of Configurables set to default values - switch-on the overwriteAxisRangeForPbPb flag, to rearrange multiplicity axes for Pb-Pb !!! O2Physics PR https://github.com/AliceO2Group/O2Physics/pull/3883#pullrequestreview-1735107548 mandatory!!! - enable the processing only for PV contributor tracks Co-authored-by: Mattia Faggin --- .../pbpb/analysis-testing-data.json | 59 +------------------ .../pbpb/analysis-testing-data.json | 59 +------------------ .../pp/analysis-testing-data.json | 58 +----------------- .../default/pbpb/analysis-testing-data.json | 59 +------------------ .../default/pbpb/analysis-testing-mc.json | 59 +------------------ .../default/pp/analysis-testing-data.json | 58 +----------------- .../json/default/pp/analysis-testing-mc.json | 58 +----------------- 7 files changed, 18 insertions(+), 392 deletions(-) diff --git a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json index c138798fe..d71d7a628 100644 --- a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json @@ -270,67 +270,14 @@ } }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", + "overwriteAxisRangeForPbPb": "true", "processData": "true", - "processDataIU": "false", + "processDataIU": "true", "processDataIUFiltered": "false", "processMC": "false", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "qa-k0s-tracking-efficiency": { diff --git a/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json index 17ca41a62..4f04fdd41 100644 --- a/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventTrackQA/pbpb/analysis-testing-data.json @@ -11,67 +11,14 @@ "syst": "pp" }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", + "overwriteAxisRangeForPbPb": "true", "processData": "true", "processDataIU": "true", - "processDataIUFiltered": "true", + "processDataIUFiltered": "false", "processMC": "false", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "track-propagation": { diff --git a/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json index 17ca41a62..d8720af92 100644 --- a/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventTrackQA/pp/analysis-testing-data.json @@ -11,67 +11,13 @@ "syst": "pp" }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", "processData": "true", "processDataIU": "true", - "processDataIUFiltered": "true", + "processDataIUFiltered": "false", "processMC": "false", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "track-propagation": { diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json index ba6042a73..a1c192b40 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json @@ -269,67 +269,14 @@ } }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", + "overwriteAxisRangeForPbPb": "true", "processData": "true", - "processDataIU": "false", + "processDataIU": "true", "processDataIUFiltered": "false", "processMC": "false", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "qa-k0s-tracking-efficiency": { diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json index 844fdebb6..deeee5c89 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json @@ -1029,67 +1029,14 @@ } }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", - "processData": "true", + "checkOnlyPVContributor": "true", + "overwriteAxisRangeForPbPb": "true", + "processData": "false", "processDataIU": "false", "processDataIUFiltered": "false", "processMC": "true", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "table-maker-m-c": { diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json index ba6042a73..58f8cf022 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -269,67 +269,13 @@ } }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", "processData": "true", - "processDataIU": "false", + "processDataIU": "true", "processDataIUFiltered": "false", "processMC": "false", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "qa-k0s-tracking-efficiency": { diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json index 844fdebb6..6c20c3dc9 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -1029,67 +1029,13 @@ } }, "qa-event-track": { - "binsPt": { - "values": [ - "0", - "0", - "0.10000000000000001", - "0.20000000000000001", - "0.29999999999999999", - "0.40000000000000002", - "0.5", - "0.59999999999999998", - "0.69999999999999996", - "0.80000000000000004", - "0.90000000000000002", - "1", - "1.1000000000000001", - "1.2", - "1.3", - "1.3999999999999999", - "1.5", - "2", - "5", - "10", - "20", - "50" - ] - }, - "binsVertexPosXY": { - "values": [ - "500", - "-1", - "1" - ] - }, - "binsVertexPosZ": { - "values": [ - "100", - "-20", - "20" - ] - }, - "fractionOfSampledEvents": "1", - "isRun3": "true", - "maxEta": "2", - "maxPhi": "10", - "maxPt": "1e+10", - "minEta": "-2", - "minPhi": "-1", - "minPt": "-10", + "checkOnlyPVContributor": "true", "processData": "true", - "processDataIU": "false", + "processDataIU": "true", "processDataIUFiltered": "false", "processMC": "true", "processTableData": "false", "processTableMC": "false", - "selectCharge": "0", - "selectGoodEvents": "true", - "selectMaxVtxZ": "100", - "selectPID": "0", - "selectPrim": "false", - "selectSec": "false", - "targetNumberOfEvents": "10000000", "trackSelection": "1" }, "table-maker-m-c": { From 4ca32ef4378a845e6cdd35cde224527c04eaa89e Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 17 Nov 2023 14:32:22 +0100 Subject: [PATCH 0207/1239] Protect against 0 or negative interaction rate --- MC/bin/o2dpg_sim_workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 0fa1067aa..a81279082 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -578,9 +578,11 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): exit(1) # Determine interation rate - # it should be taken from CDB, meanwhile some default values signalprefix='sgn_' + str(tf) INTRATE=int(args.interactionRate) + if INTRATE <= 0: + print('o2dpg_sim_workflow: Error! Interaction rate not >0 !!!') + exit(1) BCPATTERN=args.bcPatternFile includeQED = (COLTYPE == 'PbPb' or (doembedding and COLTYPEBKG == "PbPb")) or (args.with_qed == True) From 0675f2397c16c8675d81b43e7e88d3c14df5b3d4 Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 17 Nov 2023 15:17:48 +0100 Subject: [PATCH 0208/1239] Improvements in MC anchoring * Collision system is determined from GRPLHCIF * Collision energy is determined from GRPLHCIF * interaction rate calculation is done differentially as a function of collision system TODO: verify calculation and coefficients for interaction rate calculation --- MC/bin/o2dpg_sim_workflow_anchored.py | 74 ++++++++++++++++++++------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 027d6631e..f1b555028 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -206,7 +206,14 @@ def retrieve_GRP(ccdbreader, timestamp): ts, grp = ccdbreader.fetch(grp_path, "o2::parameters::GRPObject", timestamp = timestamp) return grp -def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime, ft0_eff): +def retrieve_GRPLHCIF(ccdbreader, timestamp): + """ + retrieves the GRPLHCIF object for a given time stamp + """ + _, grplhcif = ccdbreader.fetch("GLO/Config/GRPLHCIF", "o2::parameters::GRPLHCIFData", timestamp = timestamp) + return grplhcif + +def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime, ft0_eff, NBunches, ColSystem): """ retrieves the CTP scalers object for a given timestamp and run_number and calculates the interation rate to be applied in Monte Carlo digitizers @@ -215,16 +222,26 @@ def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime ts, ctpscaler = ccdbreader.fetch(path, "o2::ctp::CTPRunScalers", timestamp = timestamp) if ctpscaler != None: ctpscaler.convertRawToO2() - rate = ctpscaler.getRateGivenT(finaltime,0,0) # the simple FT0 rate from the counters - # print("Global rate " + str(rate.first) + " local rate " + str(rate.second)) - - # now get the bunch filling object which is part of GRPLHCIF and calculate - # true rate (input from Chiara Zampolli) - ts, grplhcif = ccdbreader.fetch("GLO/Config/GRPLHCIF", "o2::parameters::GRPLHCIFData", timestamp = timestamp) - coll_bunches = grplhcif.getBunchFilling().getNBunches() - mu = - math.log(1. - rate.first / 11245 / coll_bunches) / ft0_eff - finalRate = coll_bunches * mu * 11245 - return finalRate + # this is the default for pp + ctpclass = 0 # <---- we take the scaler for FT0 + ctptype = 0 + # this is the default for PbPb + if ColSystem == "PbPb": + ctpclass = 25 # <--- we take scalers for ZDC + ctptype = 7 + print("Fetching rate with class " + str(ctpclass) + " type " + str(ctptype)) + rate = ctpscaler.getRateGivenT(finaltime, ctpclass, ctptype) + #if ColSystem == "PbPb": + # rate.first = rate.first / 28. + # rate.second = rate.second / 28. + + print("Global rate " + str(rate.first) + " local rate " + str(rate.second)) + if rate.first >= 0: + # calculate true rate (input from Chiara Zampolli) using number of bunches + coll_bunches = NBunches + mu = - math.log(1. - rate.second / 11245 / coll_bunches) / ft0_eff + finalRate = coll_bunches * mu * 11245 + return finalRate print (f"[ERROR]: Could not determine interaction rate; Some (external) default used") return None @@ -277,8 +294,6 @@ def main(): parser.add_argument("--ccdb-IRate", type=bool, help="whether to try fetching IRate from CCDB/CTP", default=True) parser.add_argument("--ft0-eff", type=float, dest="ft0_eff", help="FT0 eff needed for IR", default=-1.0) parser.add_argument('forward', nargs=argparse.REMAINDER) # forward args passed to actual workflow creation - parser.add_argument("-eCM", type=float, dest="eCM", help="Energy", default=13600) - parser.add_argument("-col", type=str, dest="col", help="Collision System", default="pp") args = parser.parse_args() # split id should not be larger than production id @@ -306,6 +321,27 @@ def main(): currenttime = GLOparams["SOR"] + prod_offset * GLOparams["OrbitsPerTF"] * LHCOrbitMUS // 1000 # timestamp in milliseconds print ("Production put at time : " + str(currenttime)) + # retrieve the GRPHCIF object + grplhcif = retrieve_GRPLHCIF(ccdbreader, int(currenttime)) + eCM = grplhcif.getSqrtS() + A1 = grplhcif.getAtomicNumberB1() + A2 = grplhcif.getAtomicNumberB2() + + # determine collision system and energy + print ("Determined eMC ", eCM) + print ("Determined atomic number A1 ", A1) + print ("Determined atomic number A2 ", A2) + ColSystem = "" + if A1 == 82 and A2 == 82: + ColSystem = "PbPb" + elif A1 == 1 and A2 == 1: + ColSystem = "pp" + else: + print ("Unknown collision system ... exiting") + exit (1) + + print ("Collision system ", ColSystem) + forwardargs = " ".join([ a for a in args.forward if a != '--' ]) # retrieve interaction rate rate = None @@ -313,19 +349,19 @@ def main(): if args.ccdb_IRate == True: effT0 = args.ft0_eff if effT0 < 0: - if args.col == "pp": - if args.eCM < 1000: + if ColSystem == "pp": + if eCM < 1000: effT0 = 0.68 - elif args.eCM < 6000: + elif eCM < 6000: effT0 = 0.737 else: effT0 = 0.759 - elif args.col == "PbPb": + elif ColSystem == "PbPb": effT0 = 4.0 else: effT0 = 0.759 - rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effT0) + rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effT0, grplhcif.getBunchFilling().getNBunches(), ColSystem) if rate != None: # if the rate calculation was successful we will use it, otherwise we fall back to some rate given as part @@ -338,7 +374,7 @@ def main(): # we finally pass forward to the unanchored MC workflow creation # TODO: this needs to be done in a pythonic way clearly - forwardargs += " -tf " + str(args.tf) + " --sor " + str(GLOparams["SOR"]) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(GLOparams["FirstOrbit"]) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(args.col) + " -eCM " + str(args.eCM) + forwardargs += " -tf " + str(args.tf) + " --sor " + str(GLOparams["SOR"]) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(GLOparams["FirstOrbit"]) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) print ("forward args ", forwardargs) cmd = "${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py " + forwardargs print ("Creating time-anchored workflow...") From 6364f75e84f8a8e4aa8c3582dd5f716b5c4eaf7c Mon Sep 17 00:00:00 2001 From: noferini <9963644+noferini@users.noreply.github.com> Date: Mon, 20 Nov 2023 22:07:09 +0100 Subject: [PATCH 0209/1239] generalize trigger eff --- MC/bin/o2dpg_sim_workflow_anchored.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index f1b555028..fed5ded96 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -213,7 +213,7 @@ def retrieve_GRPLHCIF(ccdbreader, timestamp): _, grplhcif = ccdbreader.fetch("GLO/Config/GRPLHCIF", "o2::parameters::GRPLHCIFData", timestamp = timestamp) return grplhcif -def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime, ft0_eff, NBunches, ColSystem): +def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime, trig_eff, NBunches, ColSystem): """ retrieves the CTP scalers object for a given timestamp and run_number and calculates the interation rate to be applied in Monte Carlo digitizers @@ -239,7 +239,7 @@ def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime if rate.first >= 0: # calculate true rate (input from Chiara Zampolli) using number of bunches coll_bunches = NBunches - mu = - math.log(1. - rate.second / 11245 / coll_bunches) / ft0_eff + mu = - math.log(1. - rate.second / 11245 / coll_bunches) / trig_eff finalRate = coll_bunches * mu * 11245 return finalRate @@ -292,7 +292,7 @@ def main(): parser.add_argument("--split-id", type=int, help="The split id of this job within the whole production --prod-split)", default=0) parser.add_argument("-tf", type=int, help="number of timeframes per job", default=1) parser.add_argument("--ccdb-IRate", type=bool, help="whether to try fetching IRate from CCDB/CTP", default=True) - parser.add_argument("--ft0-eff", type=float, dest="ft0_eff", help="FT0 eff needed for IR", default=-1.0) + parser.add_argument("--trig-eff", type=float, dest="trig_eff", help="Trigger eff needed for IR", default=-1.0) parser.add_argument('forward', nargs=argparse.REMAINDER) # forward args passed to actual workflow creation args = parser.parse_args() @@ -347,21 +347,21 @@ def main(): rate = None if args.ccdb_IRate == True: - effT0 = args.ft0_eff - if effT0 < 0: + effTrigger = args.trig_eff + if effTrigger < 0: if ColSystem == "pp": if eCM < 1000: - effT0 = 0.68 + effTrigger = 0.68 elif eCM < 6000: - effT0 = 0.737 + effTrigger = 0.737 else: - effT0 = 0.759 + effTrigger = 0.759 elif ColSystem == "PbPb": - effT0 = 4.0 + effTrigger = 28.0 # this is ZDC else: - effT0 = 0.759 + effTrigger = 0.759 - rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effT0, grplhcif.getBunchFilling().getNBunches(), ColSystem) + rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) if rate != None: # if the rate calculation was successful we will use it, otherwise we fall back to some rate given as part From 253ed1541c0b8fd519f4e23a4336217bb5e797d4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Nov 2023 13:37:03 +0100 Subject: [PATCH 0210/1239] calib-workflow: set correct lifetimes of input / output calib proxies --- DATA/common/setenv_calib.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index edcd4f28f..affd1e2ad 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -380,12 +380,15 @@ get_proxy_connection() if [[ $3 == "timeframe" ]]; then PROXY_CONN+=" --environment DPL_OUTPUT_PROXY_ORDERED=1" elif [[ $3 == "sporadic" ]]; then - PROXY_CONN+=" --environment \"DPL_OUTPUT_PROXY_WHENANY=1 DPL_DONT_DROP_OLD_TIMESLICE=1\"" + PROXY_CONN+=" --environment \"DPL_OUTPUT_PROXY_WHENANY=1 DPL_DONT_DROP_OLD_TIMESLICE=1\" --sporadic-inputs" else echo "invalid option $3, must be (sporadic|timeframe)" 1>&2 exit 1 fi fi + if [[ $2 == "input" && $3 == "sporadic" ]]; then + PROXY_CONN+=" --sporadic-outputs" + fi if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then echo PROXY_CONN = $PROXY_CONN 1>&2 fi From e750df53d934f94c87c1f599a1eda8fc54a6b623 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Nov 2023 13:37:49 +0100 Subject: [PATCH 0211/1239] calib-workflow: bugfix: zdc calib is sporadic, since it does not send every TF --- DATA/common/setenv_calib.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index affd1e2ad..87fe461ea 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -293,11 +293,6 @@ fi # define spec for proxy for TF-based outputs from forward detectors if [[ -z ${CALIBDATASPEC_FORWARD_TF:-} ]]; then - # ZDC - if [[ $CALIB_ZDC_TDC == 1 ]]; then - add_semicolon_separated CALIBDATASPEC_FORWARD_TF "tdcZDC:ZDC/TDCCALIBDATA/0" - add_semicolon_separated CALIBDATASPEC_FORWARD_TF "histoZDC:ZDC/TDC_1DH" - fi # FT0 if [[ $CALIB_FT0_TIMEOFFSET == 1 ]]; then add_semicolon_separated CALIBDATASPEC_FORWARD_TF "timeSpectraFT0:FT0/TIME_SPECTRA/0" @@ -316,6 +311,11 @@ if [[ -z ${CALIBDATASPEC_FORWARD_SPORADIC:-} ]]; then if [[ $CALIB_FDD_INTEGRATEDCURR == 1 ]]; then add_semicolon_separated CALIBDATASPEC_FORWARD_SPORADIC "integrCurrFDD:FDD/IFDDC/0" fi + # ZDC + if [[ $CALIB_ZDC_TDC == 1 ]]; then + add_semicolon_separated CALIBDATASPEC_FORWARD_SPORADIC "tdcZDC:ZDC/TDCCALIBDATA/0" + add_semicolon_separated CALIBDATASPEC_FORWARD_SPORADIC "histoZDC:ZDC/TDC_1DH" + fi fi if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then From 51e53a2e4284a1d09c067648865b7b69039b8671 Mon Sep 17 00:00:00 2001 From: Francesco Noferini Date: Thu, 23 Nov 2023 16:39:09 +0100 Subject: [PATCH 0212/1239] fixing missing inputs in TRD digit (#1343) Co-authored-by: noferini <9963644+noferini@users.noreply.github.com> --- MC/config/QC/json/trd-digits-task.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/QC/json/trd-digits-task.json b/MC/config/QC/json/trd-digits-task.json index 301e78427..601089072 100644 --- a/MC/config/QC/json/trd-digits-task.json +++ b/MC/config/QC/json/trd-digits-task.json @@ -39,7 +39,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD" + "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" }, "taskParameters": { "peakregionstart": "7.0", From 174c4c105e5e339110ede7a810d28ed158298e2a Mon Sep 17 00:00:00 2001 From: cterrevo Date: Fri, 24 Nov 2023 10:55:26 +0100 Subject: [PATCH 0213/1239] fixing removal/renaming of files (#1337) configs for HF/D2H MC production tests with different configurations (gap/charm/beauty) --- ...bar.ini => GeneratorHF_D2H_bbbar_gap5.ini} | 2 +- ... GeneratorHF_D2H_ccbar_and_bbbar_gap3.ini} | 0 .../GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini | 8 ++ ...atorHF_D2H_ccbar_and_bbbar_gap5_Mode2.ini} | 2 +- .../GeneratorHF_D2H_ccbar_and_bbbar_gap8.ini | 8 ++ ...bar.ini => GeneratorHF_D2H_ccbar_gap5.ini} | 2 +- ...H_bbbar.C => GeneratorHF_D2H_bbbar_gap5.C} | 2 +- ...=> GeneratorHF_D2H_ccbar_and_bbbar_gap3.C} | 0 ...=> GeneratorHF_D2H_ccbar_and_bbbar_gap5.C} | 2 +- ...neratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.C | 128 ++++++++++++++++++ .../GeneratorHF_D2H_ccbar_and_bbbar_gap8.C | 128 ++++++++++++++++++ ...H_ccbar.C => GeneratorHF_D2H_ccbar_gap5.C} | 2 +- 12 files changed, 278 insertions(+), 6 deletions(-) rename MC/config/PWGHF/ini/{GeneratorHF_D2H_bbbar.ini => GeneratorHF_D2H_bbbar_gap5.ini} (84%) rename MC/config/PWGHF/ini/{GeneratorHF_D2H_ccbar_and_bbbar.ini => GeneratorHF_D2H_ccbar_and_bbbar_gap3.ini} (100%) create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini rename MC/config/PWGHF/ini/{GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini => GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.ini} (83%) create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8.ini rename MC/config/PWGHF/ini/{GeneratorHF_D2H_ccbar.ini => GeneratorHF_D2H_ccbar_gap5.ini} (84%) rename MC/config/PWGHF/ini/tests/{GeneratorHF_D2H_bbbar.C => GeneratorHF_D2H_bbbar_gap5.C} (98%) rename MC/config/PWGHF/ini/tests/{GeneratorHF_D2H_ccbar_and_bbbar.C => GeneratorHF_D2H_ccbar_and_bbbar_gap3.C} (100%) rename MC/config/PWGHF/ini/tests/{GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C => GeneratorHF_D2H_ccbar_and_bbbar_gap5.C} (98%) create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C rename MC/config/PWGHF/ini/tests/{GeneratorHF_D2H_ccbar.C => GeneratorHF_D2H_ccbar_gap5.C} (98%) diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar_gap5.ini similarity index 84% rename from MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini rename to MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar_gap5.ini index 5dcb3f77f..8a16c8f1f 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_bbbar_gap5.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredBeauty(3, -1.5, 1.5) +funcName=GeneratorPythia8GapTriggeredBeauty(5, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3.ini similarity index 100% rename from MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar.ini rename to MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3.ini diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini new file mode 100755 index 000000000..d8e55c163 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(5, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.ini similarity index 83% rename from MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini rename to MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.ini index 1a9c4bb19..ab9c6ba34 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(3, -1.5, 1.5) +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(5, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8.ini new file mode 100755 index 000000000..55eee4af3 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(8, -1.5, 1.5) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap5.ini similarity index 84% rename from MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini rename to MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap5.ini index a0aaba1cb..1b347ea16 100755 --- a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar.ini +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_gap5.ini @@ -1,7 +1,7 @@ ### The external generator derives from GeneratorPythia8. [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C -funcName=GeneratorPythia8GapTriggeredCharm(3, -1.5, 1.5) +funcName=GeneratorPythia8GapTriggeredCharm(5, -1.5, 1.5) [GeneratorPythia8] config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar_gap5.C similarity index 98% rename from MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C rename to MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar_gap5.C index 66af901cc..4a1d2d7d7 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_bbbar_gap5.C @@ -3,7 +3,7 @@ int External() std::string path{"o2sim_Kine.root"}; int checkPdgQuark{5}; - float ratioTrigger = 1./3; // one event triggered out of 3 + float ratioTrigger = 1./5; // one event triggered out of 5 std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3.C similarity index 100% rename from MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar.C rename to MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3.C diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5.C similarity index 98% rename from MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C rename to MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5.C index 79321e058..90d850360 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_Mode2.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5.C @@ -3,7 +3,7 @@ int External() { int checkPdgQuarkOne{4}; int checkPdgQuarkTwo{5}; - float ratioTrigger = 1./3; // one event triggered out of 3 + float ratioTrigger = 1./5; // one event triggered out of 5 std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.C new file mode 100644 index 000000000..90d850360 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2.C @@ -0,0 +1,128 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./5; // one event triggered out of 5 + + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C new file mode 100644 index 000000000..efc06b9db --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C @@ -0,0 +1,128 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./8; // one event triggered out of 8 + + std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {411, {{-321, 211, 211}, {-313, 211}, {211, 311}, {211, 333}}}, // D+ + {421, {{-321, 211}, {-321, 111, 211}}}, // D0 + {431, {{211, 333}, {-313, 321}}}, // Ds+ + {4122, {{-313, 2212}, {-321, 2224}, {211, 3124}, {-321, 211, 2212}, {311, 2212}}}, // Lc+ + {4132, {{211, 3312}}}, // Xic0 + {4232, {{-313, 2212}, {-321, 3324}, {211, 211, 3312}, {-321, 211, 2212}}}, // Xic+ + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.9 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.2) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_gap5.C similarity index 98% rename from MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C rename to MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_gap5.C index 3893280f7..b56b3db5c 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_gap5.C @@ -2,7 +2,7 @@ int External() { std::string path{"o2sim_Kine.root"}; int checkPdgQuark{4}; - float ratioTrigger = 1./3; // one event triggered out of 3 + float ratioTrigger = 1./5; // one event triggered out of 5 std::vector checkPdgHadron{411, 421, 431, 4122, 4132, 4232, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters From e310edaed0208e73a7b541d9e4036c6b818e273f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Mon, 27 Nov 2023 10:22:02 +0100 Subject: [PATCH 0214/1239] Add ropes (#1345) * Add ropes * Update GeneratorLFStrangenessTriggered.ini --- .../ini/GeneratorLFStrangenessTriggered.ini | 3 ++- .../pythia8/generator/pythia8_inel_136tev.cfg | 4 +-- .../generator/pythia8_inel_ropes_136tev.cfg | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini index d558289b0..ce6e95771 100644 --- a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered.ini @@ -5,7 +5,8 @@ funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/st [GeneratorPythia8] # if triggered then this will be used as the background event # config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg -config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +; config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg [DecayerPythia8] config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg b/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg index 9f11de75b..1a8453147 100644 --- a/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +++ b/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg @@ -7,8 +7,8 @@ Beams:eCM = 13600. # GeV SoftQCD:inelastic = on # all inelastic processes ### decays -ParticleDecays:limitTau0 on -ParticleDecays:tau0Max 10. +ParticleDecays:limitTau0 = on +ParticleDecays:tau0Max = 10. ### phase space cuts PhaseSpace:pTHatMin = 0.000000 diff --git a/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg b/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg new file mode 100644 index 000000000..e287ec4ce --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg @@ -0,0 +1,26 @@ +### beams +Beams:idA = 2212 # proton +Beams:idB = 2212 # proton +Beams:eCM = 13600. # GeV + +### processes +SoftQCD:inelastic = on # all inelastic processes + +### decays +ParticleDecays:limitTau0 = on +ParticleDecays:tau0Max = 10. + +### phase space cuts +PhaseSpace:pTHatMin = 0.000000 +PhaseSpace:pTHatMax = -1.000000 + +Random:setSeed = on +Random:seed = 0 + +Ropewalk:RopeHadronization = on +Ropewalk:doShoving = off +Ropewalk:doFlavour = on +Ropewalk:r0 = 0.5 +Ropewalk:m0 = 0.2 +Ropewalk:beta = 0.1 +PartonVertex:setVertex = on \ No newline at end of file From 142bcc74589f10def347f70ace5415faf34f2914 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Thu, 23 Nov 2023 14:56:54 +0100 Subject: [PATCH 0215/1239] Updating setenv_extra to run correctly with CTP options for TPC corrections --- .../configurations/asyncReco/setenv_extra.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 82006fc5d..78351b1ab 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -322,7 +322,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then echo "Using CTP inst lumi stored in data" export TPC_CORR_SCALING+=" --lumi-type 1 " elif [[ $INST_IR_FOR_TPC == "IDCCCDB" ]]; then - echo "TPC correction with IDC from CCDB will ne used" + echo "TPC correction with IDC from CCDB will be used" export TPC_CORR_SCALING+=" --lumi-type 2 " else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" @@ -333,13 +333,14 @@ elif [[ $ALIGNLEVEL == 1 ]]; then export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$ALIEN_JDL_MEANIRFORTPC" fi - if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]]; then + if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]] && ([[ -z $INST_IR_FOR_TPC ]] || [[ $INST_IR_FOR_TPC == "CTP" ]]); then + echo "We are in PbPb 2023, the default - for now - is to use CTP in the data" unset TPC_CORR_SCALING - export TPC_CORR_SCALING=";TPCCorrMap.lumiInstFactor=2.414 --lumi-type 1" + export TPC_CORR_SCALING=";TPCCorrMap.lumiInstFactor=2.414;TPCCorrMap.lumiMean=0 --lumi-type 1 " if [[ $SCALE_WITH_ZDC == 0 ]]; then # scaling with FT0 if [[ $SCALE_WITH_FT0 == 1 ]]; then - export TPC_CORR_SCALING=" --ctp-lumi-source 1 --lumi-type 1 TPCCorrMap.lumiInstFactor=135." + export TPC_CORR_SCALING=" --ctp-lumi-source 1 --lumi-type 1 TPCCorrMap.lumiInstFactor=135.;TPCCorrMap.lumiMean=0" else echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." return 1 @@ -347,6 +348,9 @@ elif [[ $ALIGNLEVEL == 1 ]]; then fi fi + echo "Final setting for TPC scaling is:" + echo $TPC_CORR_SCALING + if [[ $PERIOD != @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]] ; then echo "Setting TPCCLUSTERTIMESHIFT to 0" TPCCLUSTERTIMESHIFT=0 From 85d453657b54786b1997a6422cd8e68e7dede8b0 Mon Sep 17 00:00:00 2001 From: gvolpe Date: Wed, 22 Nov 2023 18:16:33 +0100 Subject: [PATCH 0216/1239] Adding async QC for HMPID --- DATA/production/qc-async/hmp_clusters.json | 59 ++++++++++++++++++++ DATA/production/qc-async/hmp_matchinfos.json | 59 ++++++++++++++++++++ DATA/production/qc-workflow.sh | 2 + 3 files changed, 120 insertions(+) create mode 100644 DATA/production/qc-async/hmp_clusters.json create mode 100644 DATA/production/qc-async/hmp_matchinfos.json diff --git a/DATA/production/qc-async/hmp_clusters.json b/DATA/production/qc-async/hmp_clusters.json new file mode 100644 index 000000000..3ca8941b4 --- /dev/null +++ b/DATA/production/qc-async/hmp_clusters.json @@ -0,0 +1,59 @@ +{ + "qc":{ + "config":{ + "database":{ + "implementation":"CCDB", + "host":"ccdb-test.cern.ch:8080", + "username":"not_applicable", + "password":"not_applicable", + "name":"not_applicable" + }, + "Activity":{ + "number":"42", + "type":"2" + }, + "monitoring":{ + "url":"infologger:///debug?qc" + }, + "consul":{ + "url":"http://consul-test.cern.ch:8500" + }, + "conditionDB":{ + "url":"ccdb-test.cern.ch:8080" + } + }, + "tasks":{ + "TaskClusters":{ + "active":"true", + "className":"o2::quality_control_modules::hmpid::HmpidTaskClusters", + "moduleName":"QcHMPID", + "detectorName":"HMP", + "cycleDurationSeconds":"10", + "maxNumberCycles":"-1", + "dataSource":{ + "type":"dataSamplingPolicy", + "name":"hmp-clusters" + }, + "location":"remote" + } + } + }, + "dataSamplingPolicies":[ + { + "id":"hmp-clusters", + "active":"true", + "machines":[ + + ], + "query":"intrecord:HMP/INTRECORDS1/0;clusters:HMP/CLUSTERS/0", + "samplingConditions":[ + { + "condition":"random", + "fraction":"0.1", + "seed":"0" + } + ], + "blocking":"false" + } + ] +} diff --git a/DATA/production/qc-async/hmp_matchinfos.json b/DATA/production/qc-async/hmp_matchinfos.json new file mode 100644 index 000000000..ae6151c62 --- /dev/null +++ b/DATA/production/qc-async/hmp_matchinfos.json @@ -0,0 +1,59 @@ +{ + "qc":{ + "config":{ + "database":{ + "implementation":"CCDB", + "host":"ccdb-test.cern.ch:8080", + "username":"not_applicable", + "password":"not_applicable", + "name":"not_applicable" + }, + "Activity":{ + "number":"42", + "type":"2" + }, + "monitoring":{ + "url":"infologger:///debug?qc" + }, + "consul":{ + "url":"http://consul-test.cern.ch:8500" + }, + "conditionDB":{ + "url":"ccdb-test.cern.ch:8080" + } + }, + "tasks":{ + "TaskMatches":{ + "active":"true", + "className":"o2::quality_control_modules::hmpid::HmpidTaskMatches", + "moduleName":"QcHMPID", + "detectorName":"HMP", + "cycleDurationSeconds":"10", + "maxNumberCycles":"-1", + "dataSource":{ + "type":"dataSamplingPolicy", + "name":"hmp-matches" + }, + "location":"remote" + } + } + }, + "dataSamplingPolicies":[ + { + "id":"hmp-matches", + "active":"true", + "machines":[ + + ], + "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0", + "samplingConditions":[ + { + "condition":"random", + "fraction":"0.1", + "seed":"0" + } + ], + "blocking":"false" + } + ] +} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 0a3ce917d..15e0d8c2e 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -168,6 +168,8 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_ITS:-}" ]] && QC_JSON_ITS=$O2DPG_ROOT/DATA/production/qc-async/its.json [[ -z "${QC_JSON_MFT:-}" ]] && QC_JSON_MFT=$O2DPG_ROOT/DATA/production/qc-async/mft.json [[ -z "${QC_JSON_TOF:-}" ]] && QC_JSON_TOF=$O2DPG_ROOT/DATA/production/qc-async/tof.json + [[ -z "${QC_JSON_HMP:-}" ]] && QC_JSON_HMP=$O2DPG_ROOT/DATA/production/qc-async/hmp_clusters.json + [[ -z "${QC_JSON_HMP:-}" ]] && QC_JSON_HMP=$O2DPG_ROOT/DATA/production/qc-async/hmp_matchinfos.json [[ -z "${QC_JSON_FT0:-}" ]] && QC_JSON_FT0=$O2DPG_ROOT/DATA/production/qc-async/ft0.json [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=$O2DPG_ROOT/DATA/production/qc-async/fv0.json [[ -z "${QC_JSON_FDD:-}" ]] && QC_JSON_FDD=$O2DPG_ROOT/DATA/production/qc-async/fdd.json From ad8add4b3f8d01b72440fb3bba9e5fb27acc9494 Mon Sep 17 00:00:00 2001 From: gvolpe Date: Thu, 23 Nov 2023 17:11:18 +0100 Subject: [PATCH 0217/1239] Using only one json file --- .../qc-async/{hmp_clusters.json => hmp.json} | 46 +++++++-------- DATA/production/qc-async/hmp_matchinfos.json | 59 ------------------- DATA/production/qc-workflow.sh | 3 +- 3 files changed, 21 insertions(+), 87 deletions(-) rename DATA/production/qc-async/{hmp_clusters.json => hmp.json} (55%) delete mode 100644 DATA/production/qc-async/hmp_matchinfos.json diff --git a/DATA/production/qc-async/hmp_clusters.json b/DATA/production/qc-async/hmp.json similarity index 55% rename from DATA/production/qc-async/hmp_clusters.json rename to DATA/production/qc-async/hmp.json index 3ca8941b4..8e066b572 100644 --- a/DATA/production/qc-async/hmp_clusters.json +++ b/DATA/production/qc-async/hmp.json @@ -22,38 +22,32 @@ "url":"ccdb-test.cern.ch:8080" } }, - "tasks":{ + "tasks": { "TaskClusters":{ "active":"true", "className":"o2::quality_control_modules::hmpid::HmpidTaskClusters", "moduleName":"QcHMPID", "detectorName":"HMP", - "cycleDurationSeconds":"10", + "cycleDurationSeconds":"60", "maxNumberCycles":"-1", "dataSource":{ - "type":"dataSamplingPolicy", - "name":"hmp-clusters" - }, - "location":"remote" - } - } - }, - "dataSamplingPolicies":[ - { - "id":"hmp-clusters", - "active":"true", - "machines":[ - - ], - "query":"intrecord:HMP/INTRECORDS1/0;clusters:HMP/CLUSTERS/0", - "samplingConditions":[ - { - "condition":"random", - "fraction":"0.1", - "seed":"0" + "type":"direct", + "query":"intrecord:HMP/INTRECORDS1/0;clusters:HMP/CLUSTERS/0" } - ], - "blocking":"false" - } - ] + }, + "TaskMatches":{ + "active":"true", + "className":"o2::quality_control_modules::hmpid::HmpidTaskMatches", + "moduleName":"QcHMPID", + "detectorName":"HMP", + "cycleDurationSeconds":"60", + "maxNumberCycles":"-1", + "dataSource":{ + "type":"direct", + "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0" + } + } + } + }, + "dataSamplingPolicies": [] } diff --git a/DATA/production/qc-async/hmp_matchinfos.json b/DATA/production/qc-async/hmp_matchinfos.json deleted file mode 100644 index ae6151c62..000000000 --- a/DATA/production/qc-async/hmp_matchinfos.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "qc":{ - "config":{ - "database":{ - "implementation":"CCDB", - "host":"ccdb-test.cern.ch:8080", - "username":"not_applicable", - "password":"not_applicable", - "name":"not_applicable" - }, - "Activity":{ - "number":"42", - "type":"2" - }, - "monitoring":{ - "url":"infologger:///debug?qc" - }, - "consul":{ - "url":"http://consul-test.cern.ch:8500" - }, - "conditionDB":{ - "url":"ccdb-test.cern.ch:8080" - } - }, - "tasks":{ - "TaskMatches":{ - "active":"true", - "className":"o2::quality_control_modules::hmpid::HmpidTaskMatches", - "moduleName":"QcHMPID", - "detectorName":"HMP", - "cycleDurationSeconds":"10", - "maxNumberCycles":"-1", - "dataSource":{ - "type":"dataSamplingPolicy", - "name":"hmp-matches" - }, - "location":"remote" - } - } - }, - "dataSamplingPolicies":[ - { - "id":"hmp-matches", - "active":"true", - "machines":[ - - ], - "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0", - "samplingConditions":[ - { - "condition":"random", - "fraction":"0.1", - "seed":"0" - } - ], - "blocking":"false" - } - ] -} diff --git a/DATA/production/qc-workflow.sh b/DATA/production/qc-workflow.sh index 15e0d8c2e..c1c471130 100755 --- a/DATA/production/qc-workflow.sh +++ b/DATA/production/qc-workflow.sh @@ -168,8 +168,7 @@ elif [[ -z ${QC_JSON_FROM_OUTSIDE:-} ]]; then [[ -z "${QC_JSON_ITS:-}" ]] && QC_JSON_ITS=$O2DPG_ROOT/DATA/production/qc-async/its.json [[ -z "${QC_JSON_MFT:-}" ]] && QC_JSON_MFT=$O2DPG_ROOT/DATA/production/qc-async/mft.json [[ -z "${QC_JSON_TOF:-}" ]] && QC_JSON_TOF=$O2DPG_ROOT/DATA/production/qc-async/tof.json - [[ -z "${QC_JSON_HMP:-}" ]] && QC_JSON_HMP=$O2DPG_ROOT/DATA/production/qc-async/hmp_clusters.json - [[ -z "${QC_JSON_HMP:-}" ]] && QC_JSON_HMP=$O2DPG_ROOT/DATA/production/qc-async/hmp_matchinfos.json + [[ -z "${QC_JSON_HMP:-}" ]] && QC_JSON_HMP=$O2DPG_ROOT/DATA/production/qc-async/hmp.json [[ -z "${QC_JSON_FT0:-}" ]] && QC_JSON_FT0=$O2DPG_ROOT/DATA/production/qc-async/ft0.json [[ -z "${QC_JSON_FV0:-}" ]] && QC_JSON_FV0=$O2DPG_ROOT/DATA/production/qc-async/fv0.json [[ -z "${QC_JSON_FDD:-}" ]] && QC_JSON_FDD=$O2DPG_ROOT/DATA/production/qc-async/fdd.json From b48f28b12ca0b612210611e71eebcef84fbcb6c1 Mon Sep 17 00:00:00 2001 From: gvolpe Date: Thu, 23 Nov 2023 21:26:44 +0100 Subject: [PATCH 0218/1239] Fixing conventions --- DATA/production/qc-async/hmp.json | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/DATA/production/qc-async/hmp.json b/DATA/production/qc-async/hmp.json index 8e066b572..ad228c9d0 100644 --- a/DATA/production/qc-async/hmp.json +++ b/DATA/production/qc-async/hmp.json @@ -23,31 +23,32 @@ } }, "tasks": { - "TaskClusters":{ + "HMPClusters":{ "active":"true", + "taskName" : "Clusters", "className":"o2::quality_control_modules::hmpid::HmpidTaskClusters", "moduleName":"QcHMPID", "detectorName":"HMP", "cycleDurationSeconds":"60", "maxNumberCycles":"-1", - "dataSource":{ - "type":"direct", - "query":"intrecord:HMP/INTRECORDS1/0;clusters:HMP/CLUSTERS/0" + "dataSource": { + "type":"direct", + "query":"intrecord:HMP/INTRECORDS1/0;clusters:HMP/CLUSTERS/0" } }, - "TaskMatches":{ + "HMPMatches":{ "active":"true", + "taskName" : "Matches", "className":"o2::quality_control_modules::hmpid::HmpidTaskMatches", "moduleName":"QcHMPID", "detectorName":"HMP", "cycleDurationSeconds":"60", "maxNumberCycles":"-1", - "dataSource":{ - "type":"direct", - "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0" - } - } - } - }, - "dataSamplingPolicies": [] + "dataSource": { + "type":"direct", + "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0" + } + } + } + } } From de14232e05918dcf09ca4932617fe562f25f47bf Mon Sep 17 00:00:00 2001 From: ddobrigk Date: Mon, 27 Nov 2023 15:59:23 +0100 Subject: [PATCH 0219/1239] Svertexer changes --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 78351b1ab..0813c565c 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -427,6 +427,12 @@ fi if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " fi +if [[ $ALIEN_JDL_EXCLUDETPCONLY == 1 ]]; then + export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=true" +fi +if [[ $ALIEN_JDL_SVERTEXER_ALL_SOURCES == 1 ]]; then + export SVERTEXING_SOURCES="ITS,TPC,ITS-TPC,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP" +fi export CONFIG_EXTRA_PROCESS_o2_primary_vertexing_workflow+=";$PVERTEXER;$VDRIFTPARAMOPTION;" From 5dabc51e1c79cde8824f2949e9bf7c22287b3a9e Mon Sep 17 00:00:00 2001 From: ddobrigk Date: Mon, 27 Nov 2023 19:29:06 +0100 Subject: [PATCH 0220/1239] Complete track list passed to svertexer --- DATA/common/setenv.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index a7e41c6a8..1f4af758c 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -199,7 +199,6 @@ done if [[ -z ${SVERTEXING_SOURCES:-} ]]; then SVERTEXING_SOURCES="$VERTEXING_SOURCES" - [[ -z ${TPC_TRACKS_SVERTEXING:-} ]] && SVERTEXING_SOURCES=$(echo $SVERTEXING_SOURCES | sed -E -e "s/(^|,)TPC(-TRD|-TOF)+//g" -e "s/,TPC,/,/") fi # this option requires well calibrated timing beween different detectors, at the moment suppress it From 89fb8e5ff846859d2f54128b3d23e2f807298488 Mon Sep 17 00:00:00 2001 From: ddobrigk Date: Mon, 27 Nov 2023 19:32:21 +0100 Subject: [PATCH 0221/1239] Update setenv_extra.sh --- DATA/production/configurations/asyncReco/setenv_extra.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 0813c565c..3f868e1f0 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -427,12 +427,12 @@ fi if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " fi -if [[ $ALIEN_JDL_EXCLUDETPCONLY == 1 ]]; then +# allow usage of TPC-only in svertexer (default: do not) +if [[ $ALIEN_JDL_USETPCONLY == 1 ]]; then + export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=false" +else export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=true" fi -if [[ $ALIEN_JDL_SVERTEXER_ALL_SOURCES == 1 ]]; then - export SVERTEXING_SOURCES="ITS,TPC,ITS-TPC,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP" -fi export CONFIG_EXTRA_PROCESS_o2_primary_vertexing_workflow+=";$PVERTEXER;$VDRIFTPARAMOPTION;" From 02363aca25267708df461fcd3405136ac5b15e89 Mon Sep 17 00:00:00 2001 From: ddobrigk Date: Mon, 27 Nov 2023 22:16:09 +0100 Subject: [PATCH 0222/1239] Update setenv_extra.sh --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 3f868e1f0..5f2ca3766 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -428,7 +428,7 @@ if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " fi # allow usage of TPC-only in svertexer (default: do not) -if [[ $ALIEN_JDL_USETPCONLY == 1 ]]; then +if [[ $ALIEN_JDL_USETPCONLYFORV0S == 1 ]]; then export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=false" else export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=true" From d76e92f0e47a9faef70150128b32fe93d3863b1e Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 28 Nov 2023 11:37:22 +0100 Subject: [PATCH 0223/1239] fix HMP async QC json --- DATA/production/qc-async/hmp.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/qc-async/hmp.json b/DATA/production/qc-async/hmp.json index ad228c9d0..03915cb13 100644 --- a/DATA/production/qc-async/hmp.json +++ b/DATA/production/qc-async/hmp.json @@ -46,7 +46,7 @@ "maxNumberCycles":"-1", "dataSource": { "type":"direct", - "query":"matchHMP:HMP/MATCHES/0;clsHMP_GLO_MCTR:HMP/MCLABELS/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0" + "query":"matchHMP:HMP/MATCHES/0;hmpidcluster:HMP/CLUSTERS/0;hmpidtriggers:HMP/INTRECORDS1/0" } } } From 9b12b2cc755bd3a466c4ed43f8c1cfba424269b1 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 28 Nov 2023 17:28:54 +0100 Subject: [PATCH 0224/1239] Allow passing list of files via another file Possible to use o2dpg_release_validation.py rel-val -i @list.txt -j ... where list.txt contains a list of files paths for the first batch. Can be used independently for both -i and -j. --- RelVal/o2dpg_release_validation.py | 36 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/RelVal/o2dpg_release_validation.py b/RelVal/o2dpg_release_validation.py index adccdc982..5e0223c6b 100755 --- a/RelVal/o2dpg_release_validation.py +++ b/RelVal/o2dpg_release_validation.py @@ -189,17 +189,43 @@ def rel_val_root(files1, files2, include_root_dirs, add_to_previous, metrics_ena in case of success, return the path to the JSON with computed metrics None otherwise """ - print("==> Process and compare 2 sets of files <==") + def get_files_from_list(list_filename): + """ + Quick helper - # prepare the output directory - if not exists(output_dir): - makedirs(output_dir) - log_file_rel_val = join(abspath(output_dir), "rel_val.log") + Extract filenames from what is listed in a given file + """ + collect_files = [] + with open(list_filename, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + collect_files.append(line) + return collect_files + + print("==> Process and compare 2 sets of files <==") # flat ROOT files to extract to and read from during RelVal; make absolute paths so we don't confuse ourselves when running e.g. ROOT macros in different directories file_1 = abspath(join(output_dir, "extracted_objects_1.root")) file_2 = abspath(join(output_dir, "extracted_objects_2.root")) + if len(files1) == 1 and files1[0][0] == "@": + files1 = get_files_from_list(files1[0]) + if not files1: + print(f"ERROR: Apparently {files1[0][1:]} contains no files to be extracted.") + return None + if len(files2) == 1 and files2[0][0] == "@": + files2 = get_files_from_list(files2[0]) + if not files2: + print(f"ERROR: Apparently {files2[0][1:]} contains no files to be extracted.") + return None + + # prepare the output directory + if not exists(output_dir): + makedirs(output_dir) + log_file_rel_val = join(abspath(output_dir), "rel_val.log") + if no_extract: # in this case we expect the input files to be what we would otherwise extract first if len(files1) != 1 or len(files2) != 1: From 85141fda72e35e91a28fb9eb74349375a12d3e8b Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 28 Nov 2023 17:30:42 +0100 Subject: [PATCH 0225/1239] Remove @ from filename --- RelVal/o2dpg_release_validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RelVal/o2dpg_release_validation.py b/RelVal/o2dpg_release_validation.py index 5e0223c6b..2f1bd375e 100755 --- a/RelVal/o2dpg_release_validation.py +++ b/RelVal/o2dpg_release_validation.py @@ -211,12 +211,12 @@ def get_files_from_list(list_filename): file_2 = abspath(join(output_dir, "extracted_objects_2.root")) if len(files1) == 1 and files1[0][0] == "@": - files1 = get_files_from_list(files1[0]) + files1 = get_files_from_list(files1[0][1:]) if not files1: print(f"ERROR: Apparently {files1[0][1:]} contains no files to be extracted.") return None if len(files2) == 1 and files2[0][0] == "@": - files2 = get_files_from_list(files2[0]) + files2 = get_files_from_list(files2[0][1:]) if not files2: print(f"ERROR: Apparently {files2[0][1:]} contains no files to be extracted.") return None From db4983326f8185dcb7c90731c71e84d6d819aaca Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Wed, 29 Nov 2023 13:57:15 +0100 Subject: [PATCH 0226/1239] Delete ratio plots after plotting --- RelVal/PlotOverlays.C | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/RelVal/PlotOverlays.C b/RelVal/PlotOverlays.C index f73557544..27a2c141f 100644 --- a/RelVal/PlotOverlays.C +++ b/RelVal/PlotOverlays.C @@ -127,6 +127,7 @@ void overlay1D(std::vector hVec, std::vector labelVec, TLegen if (i > 0) { // no ratio for the first histogram (which would simply be 1) TH1* hRatio = (TH1*)h->Clone(); + hRatio->SetDirectory(0); hRatio->Divide(h, hVec[0], 1.0, 1.0, "B"); // error option? ratios.push_back(hRatio); } @@ -194,6 +195,10 @@ void overlay1D(std::vector hVec, std::vector labelVec, TLegen auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; c.SaveAs(savePath.c_str()); c.Close(); + + for (auto& r : ratios) { + delete r; + } } // overlay 2D histograms @@ -218,8 +223,12 @@ void overlay2D(std::vector hVec1, std::vector labelVec, TLege t1->Draw(); } + std::vector ratios; + for (int i = 1; i < nHistos; i++){ auto hDiv = (TH2*)hVec[i]->Clone(Form("%s_ratio", hVec[i]->GetName())); + hDiv->SetDirectory(0); + ratios.push_back(hDiv); hDiv->SetTitle(hVec[i]->GetTitle() + TString("(" + labelVec[i] + "/"+labelVec[0]+")")); hDiv->SetStats(0); hDiv->Divide(hVec[0]); @@ -254,6 +263,10 @@ void overlay2D(std::vector hVec1, std::vector labelVec, TLege auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; c.SaveAs(savePath.c_str()); c.Close(); + + for (auto& r : ratios) { + delete r; + } } // entry point for overlay plots from ReleaseValidation.C From 0fdb5febf7bc00e5da403c82d3ce61afe1fd1b31 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Thu, 30 Nov 2023 10:28:32 +0100 Subject: [PATCH 0227/1239] [EMCAL-1037] Add CTP config input spec Input Spec for DPL CCDB backend for CTP configuration added, as the DPL CCDB backend replaced the QC CCDB backend in the task. --- DATA/production/qc-async/emc.json | 2 +- DATA/production/qc-async/emc_PbPb.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/emc.json b/DATA/production/qc-async/emc.json index 3a7d6fe8e..817ce5406 100644 --- a/DATA/production/qc-async/emc.json +++ b/DATA/production/qc-async/emc.json @@ -73,7 +73,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS" + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" }, "taskParameters": { } diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index 17a0c5577..c4311da6c 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -87,7 +87,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS" + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" }, "taskParameters": { "AliasMB" : "CMTVXTSC" From 53c9a89210deec9d57640e70191f4bf37dcf38bb Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Thu, 30 Nov 2023 12:19:30 +0100 Subject: [PATCH 0228/1239] Remove obsolete params --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 5f2ca3766..2eba6fed0 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -376,7 +376,7 @@ export ITSEXTRAERR="ITSCATrackerParam.sysErrY2[0]=$ERRIB;ITSCATrackerParam.sysEr # ad-hoc options for ITS reco workflow EXTRA_ITSRECO_CONFIG= if [[ $BEAMTYPE == "PbPb" ]]; then - EXTRA_ITSRECO_CONFIG="ITSCATrackerParam.trackletsPerClusterLimit=15.;ITSCATrackerParam.cellsPerClusterLimit=35.;ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" + EXTRA_ITSRECO_CONFIG="ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" elif [[ $BEAMTYPE == "pp" ]]; then EXTRA_ITSRECO_CONFIG="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" fi From 54c3931b5b3d9d5f42fede56e8baaf6e345e605e Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Tue, 28 Nov 2023 10:12:35 +0100 Subject: [PATCH 0229/1239] Update charm-hadron decay lenghts + force Xic and Omegac decay channels --- .../pythia8_charmhadronic_with_decays.cfg | 35 ++++++++++++------- ...ythia8_charmhadronic_with_decays_Mode2.cfg | 34 ++++++++++++------ 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg index d28e12d8b..8020bb75a 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg @@ -1,5 +1,7 @@ -### author: Fabrizio Grosa (fabrizio.grosa@cern.ch) -### since: January 2022 +### authors: Fabrizio Grosa (fabrizio.grosa@cern.ch) +### Cristina Terrevoli (cristina.terrevoli@cern.ch) +### Fabio Catalano (fabio.catalano@cern.ch) +### last update: November 2023 ### beams Beams:idA 2212 # proton @@ -13,8 +15,15 @@ SoftQCD:inelastic on # all inelastic processes ParticleDecays:limitTau0 on ParticleDecays:tau0Max 10. -### Force golden charm hadrons decay modes +# Correct decay lengths (wrong in PYTHIA8 decay table) +# Lb +5122:tau0 = 0.4390 +# Xic0 +4132:tau0 = 0.0455 +# OmegaC +4332:tau0 = 0.0803 +### Force golden charm hadrons decay modes for D2H studies ### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other 411:oneChannel = 1 0.0752 0 -321 211 211 411:addChannel = 1 0.0104 0 -313 211 @@ -32,7 +41,9 @@ ParticleDecays:tau0Max 10. 4232:addChannel = 1 0.2 0 3324 211 4232:addChannel = 1 0.2 0 3312 211 211 ### add Xic0 decays absent in PYTHIA8 decay table -4132:addChannel = 1 0.2 0 3312 211 +4132:addChannel = 1 0.0143 0 3312 211 +### add OmegaC decay absent in PYTHIA8 decay table +4332:addChannel = 1 1. 0 3334 211 ### K* -> K pi 313:onMode = off @@ -49,7 +60,14 @@ ParticleDecays:tau0Max 10. ### for Lambda_c -> Lambda(1520) K- 102134:onMode = off 102134:onIfAll = 2212 321 - +### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +3312:onMode = off +3312:onIfAll = 3122 -211 +3122:onMode = off +3122:onIfAll = 2212 -211 +### for Omega_c -> pi Omega -> pi K Lambda -> pi K pi p +3334:onMode = off +3334:onIfAll = 3122 -321 ### switch off all decay channels 411:onMode = off @@ -85,7 +103,6 @@ ParticleDecays:tau0Max 10. 4122:onIfMatch = 3124 211 ### Lambda_c -> p K pi 4122:onIfMatch = 2212 321 211 - ### Lambda_c -> pK0s 4122:onIfMatch = 2212 311 @@ -103,9 +120,3 @@ ParticleDecays:tau0Max 10. ### Omega_c -> Omega pi 4332:onIfMatch = 3334 211 - -# Correct Lb decay length (wrong in PYTHIA8 decay table) -5122:tau0 = 4.41000e-01 - -# Correct OmegaC decay length (wrong in PYTHIA8 decay table) -4332:tau0 = 0.08000000000 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg index 562c6e63f..cdaa4b923 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg @@ -1,5 +1,7 @@ -### author: Fabrizio Grosa (fabrizio.grosa@cern.ch) -### since: January 2022 +### authors: Fabrizio Grosa (fabrizio.grosa@cern.ch) +### Cristina Terrevoli (cristina.terrevoli@cern.ch) +### Fabio Catalano (fabio.catalano@cern.ch) +### last update: November 2023 ### beams Beams:idA 2212 # proton @@ -30,10 +32,16 @@ StringFlav:probQQ1toQQ0join 0.0275,0.0275,0.0275,0.0275 MultiPartonInteractions:pT0Ref 2.15 BeamRemnants:remnantMode 1 BeamRemnants:saturation 5 - -### Force golden charm hadrons decay modes for trigger studies +# Correct decay lengths (wrong in PYTHIA8 decay table) +# Lb +5122:tau0 = 0.4390 +# Xic0 +4132:tau0 = 0.0455 +# OmegaC +4332:tau0 = 0.0803 +### Force golden charm hadrons decay modes for D2H studies ### add D+ decays absent in PYTHIA8 decay table and set BRs from PDG for other 411:oneChannel = 1 0.0752 0 -321 211 211 411:addChannel = 1 0.0104 0 -313 211 @@ -51,7 +59,9 @@ BeamRemnants:saturation 5 4232:addChannel = 1 0.2 0 3324 211 4232:addChannel = 1 0.2 0 3312 211 211 ### add Xic0 decays absent in PYTHIA8 decay table -4132:addChannel = 1 0.2 0 3312 211 +4132:addChannel = 1 0.0143 0 3312 211 +### add OmegaC decay absent in PYTHIA8 decay table +4332:addChannel = 1 1. 0 3334 211 ### K* -> K pi 313:onMode = off @@ -68,6 +78,14 @@ BeamRemnants:saturation 5 ### for Lambda_c -> Lambda(1520) K- 102134:onMode = off 102134:onIfAll = 2212 321 +### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +3312:onMode = off +3312:onIfAll = 3122 -211 +3122:onMode = off +3122:onIfAll = 2212 -211 +### for Omega_c -> pi Omega -> pi K Lambda -> pi K pi p +3334:onMode = off +3334:onIfAll = 3122 -321 ### switch off all decay channels 411:onMode = off @@ -120,9 +138,3 @@ BeamRemnants:saturation 5 ### Omega_c -> Omega pi 4332:onIfMatch = 3334 211 - -# Correct Lb decay length (wrong in PYTHIA8 decay table) -5122:tau0 = 4.41000e-01 - -# Correct OmegaC decay length (wrong in PYTHIA8 decay table) -4332:tau0 = 0.08000000000 From e04c1a2595d40d9a734f81cccd56fb67f14d3fba Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Tue, 28 Nov 2023 10:55:13 +0100 Subject: [PATCH 0230/1239] Add config files for XiC and OmegaC test productions --- ...rHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.ini | 8 ++ ...rHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.ini | 8 ++ ..._ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.ini | 8 ++ ...rHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.ini | 8 ++ ...torHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.C | 122 +++++++++++++++++ ...torHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.C | 122 +++++++++++++++++ ...2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C | 123 ++++++++++++++++++ ...torHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.C | 122 +++++++++++++++++ 8 files changed, 521 insertions(+) create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.ini create mode 100755 MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.ini create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C create mode 100644 MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.C diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.ini new file mode 100755 index 000000000..7e6f26253 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(3, -1.5, 1.5, -1.5, 1.5, {4132}) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.ini new file mode 100755 index 000000000..74db32a78 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(5, -1.5, 1.5, -1.5, 1.5, {4132}) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.ini new file mode 100755 index 000000000..4886710e6 --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(5, -1.5, 1.5, -1.5, 1.5, {4132, 4332}) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.ini b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.ini new file mode 100755 index 000000000..93fdf7dee --- /dev/null +++ b/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.ini @@ -0,0 +1,8 @@ +### The external generator derives from GeneratorPythia8. +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGHF/external/generator/generator_pythia8_gaptriggered_hf.C +funcName=GeneratorPythia8GapTriggeredCharmAndBeauty(8, -1.5, 1.5, -1.5, 1.5, {4132}) + +[GeneratorPythia8] +config=${O2DPG_ROOT}/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +includePartonEvent=true diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.C new file mode 100644 index 000000000..c99426354 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap3_Mode2_XiC.C @@ -0,0 +1,122 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./3; // one event triggered out of 3 + + std::vector checkPdgHadron{4132}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {4132, {{211, 3312}}} // Xic0 + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.C new file mode 100644 index 000000000..b9b398cfe --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC.C @@ -0,0 +1,122 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./5; // one event triggered out of 5 + + std::vector checkPdgHadron{4132}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {4132, {{211, 3312}}} // Xic0 + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C new file mode 100644 index 000000000..b6b4d6010 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C @@ -0,0 +1,123 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./5; // one event triggered out of 5 + + std::vector checkPdgHadron{4132}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {4132, {{211, 3312}}}, // Xic0 + {4332, {{211, 3334}}} // Omegac+ + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.C new file mode 100644 index 000000000..596639d12 --- /dev/null +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8_Mode2_XiC.C @@ -0,0 +1,122 @@ +int External() { + std::string path{"o2sim_Kine.root"}; + + int checkPdgQuarkOne{4}; + int checkPdgQuarkTwo{5}; + float ratioTrigger = 1./8; // one event triggered out of 8 + + std::vector checkPdgHadron{4132}; + std::map>> checkHadronDecays{ // sorted pdg of daughters + {4132, {{211, 3312}}} // Xic0 + }; + + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + + auto tree = (TTree *)file.Get("o2sim"); + std::vector *tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + o2::dataformats::MCEventHeader *eventHeader = nullptr; + tree->SetBranchAddress("MCEventHeader.", &eventHeader); + + int nEventsMB{}, nEventsInjOne{}, nEventsInjTwo{}; + int nQuarksOne{}, nQuarksTwo{}, nSignals{}, nSignalGoodDecay{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + + // check subgenerator information + if (eventHeader->hasInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID)) { + bool isValid = false; + int subGeneratorId = eventHeader->getInfo(o2::mcgenid::GeneratorProperty::SUBGENERATORID, isValid); + if (subGeneratorId == 0) { + nEventsMB++; + } else if (subGeneratorId == checkPdgQuarkOne) { + nEventsInjOne++; + } else if (subGeneratorId == checkPdgQuarkTwo) { + nEventsInjTwo++; + } + } + + for (auto &track : *tracks) { + auto pdg = track.GetPdgCode(); + if (std::abs(pdg) == checkPdgQuarkOne) { + nQuarksOne++; + continue; + } + if (std::abs(pdg) == checkPdgQuarkTwo) { + nQuarksTwo++; + continue; + } + if (std::find(checkPdgHadron.begin(), checkPdgHadron.end(), std::abs(pdg)) != checkPdgHadron.end()) { // found signal + nSignals++; // count signal PDG + + std::vector pdgsDecay{}; + std::vector pdgsDecayAntiPart{}; + for (int j{track.getFirstDaughterTrackId()}; j <= track.getLastDaughterTrackId(); ++j) { + auto pdgDau = tracks->at(j).GetPdgCode(); + pdgsDecay.push_back(pdgDau); + if (pdgDau != 333) { // phi is antiparticle of itself + pdgsDecayAntiPart.push_back(-pdgDau); + } else { + pdgsDecayAntiPart.push_back(pdgDau); + } + } + + std::sort(pdgsDecay.begin(), pdgsDecay.end()); + std::sort(pdgsDecayAntiPart.begin(), pdgsDecayAntiPart.end()); + + for (auto &decay : checkHadronDecays[std::abs(pdg)]) { + if (pdgsDecay == decay || pdgsDecayAntiPart == decay) { + nSignalGoodDecay++; + break; + } + } + } + } + } + + std::cout << "--------------------------------\n"; + std::cout << "# Events: " << nEvents << "\n"; + std::cout << "# MB events: " << nEventsMB << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkOne) << nEventsInjOne << "\n"; + std::cout << Form("# events injected with %d quark pair: ", checkPdgQuarkTwo) << nEventsInjTwo << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkOne) << nQuarksOne << "\n"; + std::cout << Form("# %d (anti)quarks: ", checkPdgQuarkTwo) << nQuarksTwo << "\n"; + std::cout <<"# signal hadrons: " << nSignals << "\n"; + std::cout <<"# signal hadrons decaying in the correct channel: " << nSignalGoodDecay << "\n"; + + if (nEventsMB < nEvents * (1 - ratioTrigger) * 0.95 || nEventsMB > nEvents * (1 - ratioTrigger) * 1.05) { // we put some tolerance since the number of generated events is small + std::cerr << "Number of generated MB events different than expected\n"; + return 1; + } + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.85 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.15) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; + return 1; + } + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.85 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.15) { + std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; + return 1; + } + + if (nQuarksOne < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkOne << " lower than expected\n"; + return 1; + } + if (nQuarksTwo < nEvents * ratioTrigger) { // we expect anyway more because the same quark is repeated several time, after each gluon radiation + std::cerr << "Number of generated (anti)quarks " << checkPdgQuarkTwo << " lower than expected\n"; + return 1; + } + + float fracForcedDecays = float(nSignalGoodDecay) / nSignals; + if (fracForcedDecays < 0.9) { // we put some tolerance (e.g. due to oscillations which might change the final state) + std::cerr << "Fraction of signals decaying into the correct channel " << fracForcedDecays << " lower than expected\n"; + return 1; + } + + return 0; +} From 51963b1bd5a8258c78c1f5c8ed1d36f806ffcb45 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Tue, 28 Nov 2023 10:55:38 +0100 Subject: [PATCH 0231/1239] Update tolerances for gap-trigger 8 --- .../PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C index efc06b9db..c66f02807 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap8.C @@ -100,11 +100,11 @@ int External() { std::cerr << "Number of generated MB events different than expected\n"; return 1; } - if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.9 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.2) { + if (nEventsInjOne < nEvents * ratioTrigger * 0.5 * 0.85 || nEventsInjOne > nEvents * ratioTrigger * 0.5 * 1.15) { std::cerr << "Number of generated events injected with " << checkPdgQuarkOne << " different than expected\n"; return 1; } - if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.95 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.05) { + if (nEventsInjTwo < nEvents * ratioTrigger * 0.5 * 0.85 || nEventsInjTwo > nEvents * ratioTrigger * 0.5 * 1.15) { std::cerr << "Number of generated events injected with " << checkPdgQuarkTwo << " different than expected\n"; return 1; } From 8cf421d339fdceabb21762bcfe47e4aa6a933361 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Wed, 29 Nov 2023 11:04:28 +0100 Subject: [PATCH 0232/1239] Add OmegaC to Xi pi --- ...ratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C | 4 ++-- .../generator/pythia8_charmhadronic_with_decays.cfg | 10 +++++++--- .../pythia8_charmhadronic_with_decays_Mode2.cfg | 10 +++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C index b6b4d6010..c3fe5939c 100644 --- a/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C +++ b/MC/config/PWGHF/ini/tests/GeneratorHF_D2H_ccbar_and_bbbar_gap5_Mode2_XiC_OmegaC.C @@ -5,10 +5,10 @@ int External() { int checkPdgQuarkTwo{5}; float ratioTrigger = 1./5; // one event triggered out of 5 - std::vector checkPdgHadron{4132}; + std::vector checkPdgHadron{4132, 4332}; std::map>> checkHadronDecays{ // sorted pdg of daughters {4132, {{211, 3312}}}, // Xic0 - {4332, {{211, 3334}}} // Omegac+ + {4332, {{211, 3334}, {211, 3312}}} // Omegac+ }; TFile file(path.c_str(), "READ"); diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg index 8020bb75a..472204cbf 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays.cfg @@ -42,8 +42,9 @@ ParticleDecays:tau0Max 10. 4232:addChannel = 1 0.2 0 3312 211 211 ### add Xic0 decays absent in PYTHIA8 decay table 4132:addChannel = 1 0.0143 0 3312 211 -### add OmegaC decay absent in PYTHIA8 decay table -4332:addChannel = 1 1. 0 3334 211 +### add OmegaC decays absent in PYTHIA8 decay table +4332:addChannel = 1 0.5 0 3334 211 +4332:addChannel = 1 0.5 0 3312 211 ### K* -> K pi 313:onMode = off @@ -60,7 +61,8 @@ ParticleDecays:tau0Max 10. ### for Lambda_c -> Lambda(1520) K- 102134:onMode = off 102134:onIfAll = 2212 321 -### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +### and Omega_c -> pi Xi -> pi pi Lambda -> pi pi pi p 3312:onMode = off 3312:onIfAll = 3122 -211 3122:onMode = off @@ -120,3 +122,5 @@ ParticleDecays:tau0Max 10. ### Omega_c -> Omega pi 4332:onIfMatch = 3334 211 +### Omega_c -> Xi pi +4332:onIfMatch = 3312 211 diff --git a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg index cdaa4b923..885531e5b 100644 --- a/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg +++ b/MC/config/PWGHF/pythia8/generator/pythia8_charmhadronic_with_decays_Mode2.cfg @@ -60,8 +60,9 @@ BeamRemnants:saturation 5 4232:addChannel = 1 0.2 0 3312 211 211 ### add Xic0 decays absent in PYTHIA8 decay table 4132:addChannel = 1 0.0143 0 3312 211 -### add OmegaC decay absent in PYTHIA8 decay table -4332:addChannel = 1 1. 0 3334 211 +### add OmegaC decays absent in PYTHIA8 decay table +4332:addChannel = 1 0.5 0 3334 211 +4332:addChannel = 1 0.5 0 3312 211 ### K* -> K pi 313:onMode = off @@ -78,7 +79,8 @@ BeamRemnants:saturation 5 ### for Lambda_c -> Lambda(1520) K- 102134:onMode = off 102134:onIfAll = 2212 321 -### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +### for Xic0 -> pi Xi -> pi pi Lambda -> pi pi pi p +### and Omega_c -> pi Xi -> pi pi Lambda -> pi pi pi p 3312:onMode = off 3312:onIfAll = 3122 -211 3122:onMode = off @@ -138,3 +140,5 @@ BeamRemnants:saturation 5 ### Omega_c -> Omega pi 4332:onIfMatch = 3334 211 +### Omega_c -> Xi pi +4332:onIfMatch = 3312 211 From 9b991a847e348306b7f840206146611d93831768 Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Thu, 30 Nov 2023 15:18:05 +0100 Subject: [PATCH 0233/1239] ITS async PbPb reco: disable masking and enable squashing --- DATA/production/configurations/asyncReco/setenv_extra.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 2eba6fed0..64cc07505 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -77,7 +77,8 @@ echo RUN = $RUNNUMBER if [[ $RUNNUMBER -ge 521889 ]]; then export ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow="$ARGS_EXTRA_PROCESS_o2_ctf_reader_workflow --its-digits --mft-digits" export DISABLE_DIGIT_CLUSTER_INPUT="--digits-from-upstream" - MAXBCDIFFTOMASKBIAS_ITS="ITSClustererParam.maxBCDiffToMaskBias=10" + MAXBCDIFFTOMASKBIAS_ITS="ITSClustererParam.maxBCDiffToMaskBias=-10" # this explicitly disables ITS masking + MAXBCDIFFTOSQUASHBIAS_ITS="ITSClustererParam.maxBCDiffToSquashBias=10" # this explicitly enables ITS squashing MAXBCDIFFTOMASKBIAS_MFT="MFTClustererParam.maxBCDiffToMaskBias=10" fi # shift by +1 BC TRD(2), PHS(4), CPV(5), EMC(6), HMP(7) and by (orbitShift-1)*3564+1 BCs the ZDC since it internally resets the orbit to 1 at SOR and BC is shifted by -1 like for triggered detectors. @@ -380,7 +381,7 @@ if [[ $BEAMTYPE == "PbPb" ]]; then elif [[ $BEAMTYPE == "pp" ]]; then EXTRA_ITSRECO_CONFIG="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" fi -export CONFIG_EXTRA_PROCESS_o2_its_reco_workflow+=";$MAXBCDIFFTOMASKBIAS_ITS;$EXTRA_ITSRECO_CONFIG;" +export CONFIG_EXTRA_PROCESS_o2_its_reco_workflow+=";$MAXBCDIFFTOMASKBIAS_ITS;$MAXBCDIFFTOSQUASHBIAS_ITS;$EXTRA_ITSRECO_CONFIG;" # in the ALIGNLEVEL there was inconsistency between the internal errors of sync_misaligned and ITSEXTRAERR if [[ $ALIGNLEVEL != 0 ]]; then From e0c1501a4a0621dc5670001a3b2ee3cb0a4ec6fd Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Thu, 30 Nov 2023 20:10:12 +0100 Subject: [PATCH 0234/1239] MFT async PbPb reco: disable masking and enable squashing --- DATA/production/configurations/asyncReco/setenv_extra.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 64cc07505..ed0e7b3d6 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -79,7 +79,8 @@ if [[ $RUNNUMBER -ge 521889 ]]; then export DISABLE_DIGIT_CLUSTER_INPUT="--digits-from-upstream" MAXBCDIFFTOMASKBIAS_ITS="ITSClustererParam.maxBCDiffToMaskBias=-10" # this explicitly disables ITS masking MAXBCDIFFTOSQUASHBIAS_ITS="ITSClustererParam.maxBCDiffToSquashBias=10" # this explicitly enables ITS squashing - MAXBCDIFFTOMASKBIAS_MFT="MFTClustererParam.maxBCDiffToMaskBias=10" + MAXBCDIFFTOMASKBIAS_MFT="MFTClustererParam.maxBCDiffToMaskBias=-10" # this explicitly disables MFT masking + MAXBCDIFFTOSQUASHBIAS_MFT="MFTClustererParam.maxBCDiffToSquashBias=10" # this explicitly enables MFT squashing fi # shift by +1 BC TRD(2), PHS(4), CPV(5), EMC(6), HMP(7) and by (orbitShift-1)*3564+1 BCs the ZDC since it internally resets the orbit to 1 at SOR and BC is shifted by -1 like for triggered detectors. # run 520403: orbitShift = 59839744 --> final shift = 213268844053 @@ -466,9 +467,9 @@ export ARGS_EXTRA_PROCESS_o2_fv0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_fv0_reco_ # ad-hoc settings for MFT if [[ $BEAMTYPE == "pp" || $PERIOD == "LHC22s" ]]; then - export CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow+=";MFTTracking.RBins=30;MFTTracking.PhiBins=120;MFTTracking.ZVtxMin=-13;MFTTracking.ZVtxMax=13;MFTTracking.MFTRadLength=0.084;$MAXBCDIFFTOMASKBIAS_MFT" + export CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow+=";MFTTracking.RBins=30;MFTTracking.PhiBins=120;MFTTracking.ZVtxMin=-13;MFTTracking.ZVtxMax=13;MFTTracking.MFTRadLength=0.084;$MAXBCDIFFTOMASKBIAS_MFT;$MAXBCDIFFTOSQUASHBIAS_MFT" else - export CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow+=";MFTTracking.MFTRadLength=0.084;$MAXBCDIFFTOMASKBIAS_MFT" + export CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow+=";MFTTracking.MFTRadLength=0.084;$MAXBCDIFFTOMASKBIAS_MFT;$MAXBCDIFFTOSQUASHBIAS_MFT" fi # ad-hoc settings for MCH From b5ab2919d541dca955f3b46b60983ae6a9183863 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Wed, 29 Nov 2023 17:33:32 +0100 Subject: [PATCH 0235/1239] [EMCAL-1054] Move EMCAL to digitizer group serving as CTP input In addition adding CTP to list of detectors in digit task --- MC/bin/o2dpg_sim_workflow.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index a81279082..29c4a63a2 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -472,11 +472,11 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): workflow['stages'].append(BKG_HEADER_task) # a list of smaller sensors (used to construct digitization tasks in a parametrized way) -smallsensorlist = [ "ITS", "TOF", "FDD", "MCH", "MID", "MFT", "HMP", "EMC", "PHS", "CPV" ] +smallsensorlist = [ "ITS", "TOF", "FDD", "MCH", "MID", "MFT", "HMP", "PHS", "CPV" ] if args.with_ZDC: smallsensorlist += [ "ZDC" ] # a list of detectors that serve as input for the trigger processor CTP --> these need to be processed together for now -ctp_trigger_inputlist = [ "FT0", "FV0" ] +ctp_trigger_inputlist = [ "FT0", "FV0", "EMC" ] BKG_HITDOWNLOADER_TASKS={} for det in [ 'TPC', 'TRD' ] + smallsensorlist + ctp_trigger_inputlist: @@ -899,13 +899,15 @@ def createRestDigiTask(name, det='ALLSMALLER'): tneeds = [ContextTask['name']] if includeQED: tneeds += [QED_task['name']] - t = createTask(name="ft0fv0ctp_digi_" + str(tf), needs=tneeds, + t = createTask(name="ft0fv0emcctp_digi_" + str(tf), needs=tneeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') t['cmd'] = ('','ln -nfs ../bkg_HitsFT0.root . ; ln -nfs ../bkg_HitsFV0.root . ;')[doembedding] - t['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --onlyDet FT0,FV0,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' + putConfigValuesNew() + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs + t['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --onlyDet FT0,FV0,EMC,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' + putConfigValuesNew() + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs workflow['stages'].append(t) det_to_digitask["FT0"]=t det_to_digitask["FV0"]=t + det_to_digitask["EMC"]=t + det_to_digitask["CTP"]=t def getDigiTaskName(det): t = det_to_digitask.get(det) From 653ffdd44352d149939f9d823695d53d3c9ce3fe Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Thu, 30 Nov 2023 14:49:51 +0100 Subject: [PATCH 0236/1239] [EMCAL-755] Update EMCAL tasks in MC - Add cluster QC - Rename config to `emc-reco-tasks.json` --- DATA/production/qc-async/emc.json | 10 +++ DATA/production/qc-async/emc_PbPb.json | 10 +++ MC/bin/o2dpg_qc_finalization_workflow.py | 2 +- MC/bin/o2dpg_sim_workflow.py | 4 +- MC/config/QC/json/emc-cell-task.json | 45 ------------- MC/config/QC/json/emc-reco-tasks.json | 82 ++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 48 deletions(-) delete mode 100644 MC/config/QC/json/emc-cell-task.json create mode 100644 MC/config/QC/json/emc-reco-tasks.json diff --git a/DATA/production/qc-async/emc.json b/DATA/production/qc-async/emc.json index 817ce5406..5495b1961 100644 --- a/DATA/production/qc-async/emc.json +++ b/DATA/production/qc-async/emc.json @@ -61,6 +61,16 @@ "clusterizerMaxTimeDelta": "1000.", "hasInvMassMesons": "true", "mesonClustersRejectExotics": 1 + }, + "grpGeomRequest" : { + "geomRequest": "Ideal", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" } }, "BCTaskEMCAL": { diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index c4311da6c..f848c7ce5 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -75,6 +75,16 @@ "mesonClusterMaxTime": "50.", "MultiplicityRange":"400", "mesonMinPt": "7." + }, + "grpGeomRequest" : { + "geomRequest": "Ideal", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" } }, "BCTaskEMCAL": { diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 09d1e7b24..ee2af25f5 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -78,7 +78,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('mftDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-digits-0.json', MFTDigitsQCneeds) add_QC_finalization('mftClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-clusters.json') add_QC_finalization('mftTracksQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks.json') - add_QC_finalization('emcCellQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-cell-task.json') + add_QC_finalization('emcRecoQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json') add_QC_finalization('tpcStandardQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-standard-direct.json') add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-digits-task.json') diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 29c4a63a2..a398a862d 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1237,10 +1237,10 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): ### EMCAL if isActive('EMC'): - addQCPerTF(taskName='emcCellQC', + addQCPerTF(taskName='emcRecoQC', needs=[EMCRECOtask['name']], readerCommand='o2-emcal-cell-reader-workflow --infile emccells.root', - configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/emc-cell-task.json') + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') ### FT0 addQCPerTF(taskName='RecPointsQC', needs=[FT0RECOtask['name']], diff --git a/MC/config/QC/json/emc-cell-task.json b/MC/config/QC/json/emc-cell-task.json deleted file mode 100644 index b9fef5db6..000000000 --- a/MC/config/QC/json/emc-cell-task.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - } - }, - "tasks": { - "Cells": { - "active": "true", - "className": "o2::quality_control_modules::emcal::CellTask", - "moduleName": "QcEMCAL", - "detectorName": "EMC", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "emcal-cells:EMC/CELLS;emcal-triggerecords:EMC/CELLSTRGR" - } - } - }, - "checks": {} - }, - "dataSamplingPolicies": [] -} diff --git a/MC/config/QC/json/emc-reco-tasks.json b/MC/config/QC/json/emc-reco-tasks.json new file mode 100644 index 000000000..550943f55 --- /dev/null +++ b/MC/config/QC/json/emc-reco-tasks.json @@ -0,0 +1,82 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "Cells": { + "active": "true", + "className": "o2::quality_control_modules::emcal::CellTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-cells:EMC/CELLS/0;emcal-triggerecords:EMC/CELLSTRGR/0" + } + }, + "Clusters": { + "active": "true", + "taskName": "Clusters", + "className": "o2::quality_control_modules::emcal::ClusterTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-cells:EMC/CELLS/0;emcal-triggerecords:EMC/CELLSTRGR/0" + }, + "taskParameters": { + "useInternalClusterizer": "true", + "bindingCellTriggerRecords": "emcal-triggerecords", + "calibrateCells": "false", + "clusterizerSeedThreshold": "0.5", + "clusterizerCellTreshold": "0.1", + "clusterizerDoGradientCut": "true", + "clusterizerGradientCut": "0.03", + "clusterizerMinTime": "-300.", + "clusterizerMaxTime": "300.", + "clusterizerMaxTimeDelta": "1000.", + "hasInvMassMesons": "true", + "mesonClustersRejectExotics": 1 + }, + "grpGeomRequest" : { + "geomRequest": "Ideal", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + } + } + }, + "checks": {} + }, + "dataSamplingPolicies": [] +} From 179f97b6c28572d972440b6c8b0609b9e76771b7 Mon Sep 17 00:00:00 2001 From: Matteo Concas Date: Thu, 30 Nov 2023 11:57:13 +0100 Subject: [PATCH 0237/1239] Remove obsolete cuts for ITS iterations --- MC/bin/o2dpg_sim_config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index cb9987cf9..279f73b36 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -83,8 +83,6 @@ def add(cfg, flatconfig): # deal with larger combinatorics if args.col == "PbPb" or (args.embedding and args.colBkg == "PbPb"): - add(config, {"ITSCATrackerParam.trackletsPerClusterLimit": 20, - "ITSCATrackerParam.cellsPerClusterLimit": 20, - "ITSVertexerParam.lowMultBeamDistCut": "0."}) + add(config, {"ITSVertexerParam.lowMultBeamDistCut": "0."}) return config From 76c285ccc19fe102aedea68faecb2b49380e9b85 Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 1 Dec 2023 15:43:12 +0100 Subject: [PATCH 0238/1239] Utility improvements for analysis of errored GRID jobs * new script to fetch latest master-job-id given jobtype * new script to automatically extract relevant errored log files --- GRID/utils/extractErroredLogFiles.sh | 28 +++++++++++++++++++++++ GRID/utils/fetch_output_onfailure.sh | 17 ++++++++++++-- GRID/utils/querymasterjob.sh | 33 ++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100755 GRID/utils/extractErroredLogFiles.sh create mode 100755 GRID/utils/querymasterjob.sh diff --git a/GRID/utils/extractErroredLogFiles.sh b/GRID/utils/extractErroredLogFiles.sh new file mode 100755 index 000000000..39ce80e79 --- /dev/null +++ b/GRID/utils/extractErroredLogFiles.sh @@ -0,0 +1,28 @@ +# We determine the O2DPG task that failed (as listed in stdout) and extract the relevant log automatically +# Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs + +errored_tasks="" +find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*//' | tr ":" " " | while IFS= read -r line; do + stdoutpath=$(echo "$line" | awk '{print $1}') # Extracting the first column + logfile=$(echo "$line" | awk '{print $2}') # Extracting the second column + + dir=$(dirname ${stdoutpath}) + cd ${dir} + # determine a timeframe number (if applicable) + # Extracting the timeframe 'tf' + tf=${logfile#*_} # Removes everything before the first underscore + tf=${tf%.log} # Removes the ".log" extension + echo "Extracted timeframe ${tf}" + + # extract the general log archive + unzip -n log_archive.zip + # extract specific task log from debug archive + tar -xvzf debug_log_archive.tgz --wildcards "*${logfile}.log" + if [[ ${logfile} == *"qedsim"* || ${logfile} == *"sgnsim"* || ${logfile} == *"bkgsim"* ]]; then + # simulation has few more files to inspect + tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*serverlog*" + tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*workerlog*" + tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*mergerlog*" + fi + cd $OLDPWD +done diff --git a/GRID/utils/fetch_output_onfailure.sh b/GRID/utils/fetch_output_onfailure.sh index beeb7d4d0..076ef996e 100755 --- a/GRID/utils/fetch_output_onfailure.sh +++ b/GRID/utils/fetch_output_onfailure.sh @@ -1,6 +1,6 @@ #!/bin/bash -# a script which fetches error files for each failing subjob of a masterjob +# a script which fetches error files for each failing subjob of an AliEn GRID masterjob MY_JOBID=$1 if [ ! "${MY_JOBID}" ]; then @@ -10,11 +10,14 @@ fi SUBJOBIDS=($(alien.py ps --trace ${MY_JOBID} | awk '/Subjob submitted/' | sed 's/.*submitted: //' | tr '\n' ' ')) +CurrDir=${PWD} OutputDir=/tmp/AlienLogs_${MY_JOBID} # job_number=${#SUBJOBIDS[@]} + +# We fetch the error files locally for ((i = 0; i < job_number; i++)); do jobid=${SUBJOBIDS[i]} STATUS=$(alien.py ps -j ${jobid} | awk '//{print $4}') @@ -26,4 +29,14 @@ for ((i = 0; i < job_number; i++)); do echo "Recycle out is ${RecycleOutputDir}" alien.py cp ${RecycleOutputDir}/'*' file:${OutputDir}/${jobid} fi -done \ No newline at end of file +done + +echo " ... Going to automatic extraction of log files ... " + +# We determine the O2DPG task that failed (as listed in stdout) and extract the relevant log automatically +# Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs +cd ${OutputDir} +# call the extraction script +${O2DPG_ROOT}/GRID/utils/extractErroredLogFiles.sh +echo "Files have been extracted to ${OutputDir}" +cd ${CurrDir} diff --git a/GRID/utils/querymasterjob.sh b/GRID/utils/querymasterjob.sh new file mode 100755 index 000000000..2f300c52f --- /dev/null +++ b/GRID/utils/querymasterjob.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# A script which queries the (latest) masterjob given an AliEn jobtype + +MY_JOBTYPE=$1 +if [ ! "${MY_JOBTYPE}" ]; then + echo "Please provide a job type as first argument" + exit 1 +fi + +# use curl to fetch the table of jobs for this jobtype +URL="https://alimonitor.cern.ch/prod/jobs.jsp?t=${MY_JOBTYPE}&res_path=csv" +echo "Querying ${URL}" +latestjob=$(curl --cert ~/.globus/usercert.pem --key ~/.globus/userkey.pem -k -s --no-keepalive -H "Connection: close" ${URL} 2> /dev/null | grep -v "Run" | head -n 1) + +#The output format is +#PID;Run number;Packages;Output directory;Total jobs;Done jobs;Running jobs;Waiting jobs;Error jobs;Other jobs;Wall time;Saving time;Output size +# split string on ";" +IFS=';' read -r -a tokens <<< "$latestjob" +masterid=${tokens[0]} +if [ ! "${masterid}" ]; then + exit 1 +fi + +totaljobs=${tokens[4]} +donejobs=${tokens[5]} +runningjobs=${tokens[6]} +erroredjobs=${tokens[8]} +echo "MasterID $masterid" +echo "TotalJobs $totaljobs" +echo "DoneJobs $donejobs" +echo "RunningJobs $donejobs" +echo "ErroredJobs $erroredjobs" From 1a0c994d28747dbd2975559277969dd3cad66269 Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 1 Dec 2023 21:18:59 +0100 Subject: [PATCH 0239/1239] Simple shell script to detected failure reason --- GRID/utils/extractFailureMessage.sh | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 GRID/utils/extractFailureMessage.sh diff --git a/GRID/utils/extractFailureMessage.sh b/GRID/utils/extractFailureMessage.sh new file mode 100755 index 000000000..ea78ff2e7 --- /dev/null +++ b/GRID/utils/extractFailureMessage.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# script to check for indication of failure in a file +# and to report on the first error + +filetocheck=$1 +if [ ! "${filetocheck}" ]; then + echo "Specify a file to search" + exit 1 +fi + +# list of typical failure patterns (append if needed) +pattern="-e \"\<[Ee]xception\" \ + -e \"segmentation violation\" \ + -e \"error while setting up workflow\" \ + -e \"bus error\" \ + -e \"Assertion.*failed\" \ + -e \"Fatal in\" \ + -e \"libc++abi.*terminating\" \ + -e \"There was a crash.\" \ + -e \"arrow.*Check failed\" \ + -e \"terminate called after\" \ + -e \"terminate called without an active\" \ + -e \"\]\[FATAL\]\" \ + -e \"\*\*\* Program crashed\" \ + -e \"\*\*\* Error in\"" # <--- LIBC fatal error messages + +CMD="grep -m 1 -B2 -A10 ${pattern} $filetocheck" +T=$(eval "${CMD}") +if [ "$?" -eq 0 ]; then + echo "" + echo "==============================ERROR in ${filetocheck} ==============================" + echo "" + echo "${T}" +fi \ No newline at end of file From 430c761ee6ca249e2577fd34307c1b980e213f67 Mon Sep 17 00:00:00 2001 From: afurs Date: Sat, 2 Dec 2023 16:57:21 +0300 Subject: [PATCH 0240/1239] FT0: disabling slewing correction for MC --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index a398a862d..6ada53a15 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -964,7 +964,7 @@ def getDigiTaskName(det): workflow['stages'].append(ITSRECOtask) FT0RECOtask=createTask(name='ft0reco_'+str(tf), needs=[getDigiTaskName("FT0")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') - FT0RECOtask['cmd'] = '${O2_ROOT}/bin/o2-ft0-reco-workflow --disable-time-offset-calib ' + getDPL_global_options() + putConfigValues() + FT0RECOtask['cmd'] = '${O2_ROOT}/bin/o2-ft0-reco-workflow --disable-time-offset-calib --disable-slewing-calib ' + getDPL_global_options() + putConfigValues() workflow['stages'].append(FT0RECOtask) ITSTPCMATCHtask=createTask(name='itstpcMatch_'+str(tf), needs=[TPCRECOtask['name'], ITSRECOtask['name'], FT0RECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='8000', relative_cpu=3/8) From d97ac6aa9d201e86484ffa060bd1641ec2a89cb9 Mon Sep 17 00:00:00 2001 From: afurs Date: Sat, 2 Dec 2023 17:29:49 +0300 Subject: [PATCH 0241/1239] FT0: increasing amp cut for FT0 time calculation in reco, only for PbPb --- DATA/production/configurations/asyncReco/setenv_extra.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index ed0e7b3d6..b83c22fd5 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -458,6 +458,9 @@ export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPC # ad-hoc settings for FT0 export ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow --ft0-reconstructor" +if [[ $BEAMTYPE == "PbPb" ]]; then + export CONFIG_EXTRA_PROCESS_o2_ft0_reco_workflow=";FT0TimeFilterParam.mAmpLower=20;" +fi # ad-hoc settings for FV0 export ARGS_EXTRA_PROCESS_o2_fv0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_fv0_reco_workflow --fv0-reconstructor" From d23b6cc59cc4dd03ebb2233761d6f2ffe10e86f9 Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Fri, 1 Dec 2023 13:29:44 +0100 Subject: [PATCH 0242/1239] [EMCAL-1054] Enable BC task in simulations Include EMCAL BC QC in simulations in case CTP is included. --- DATA/production/qc-async/emc_PbPb.json | 2 +- MC/bin/o2dpg_qc_finalization_workflow.py | 1 + MC/bin/o2dpg_sim_workflow.py | 5 +++ MC/config/QC/json/emc-bc-task.json | 49 ++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 MC/config/QC/json/emc-bc-task.json diff --git a/DATA/production/qc-async/emc_PbPb.json b/DATA/production/qc-async/emc_PbPb.json index f848c7ce5..ce505440d 100644 --- a/DATA/production/qc-async/emc_PbPb.json +++ b/DATA/production/qc-async/emc_PbPb.json @@ -100,7 +100,7 @@ "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" }, "taskParameters": { - "AliasMB" : "CMTVXTSC" + "AliasMB" : "CMTVXTSC,C0TVXTSC,C0TVXTCE" } } } diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index ee2af25f5..415c789ca 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -79,6 +79,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('mftClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-clusters.json') add_QC_finalization('mftTracksQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks.json') add_QC_finalization('emcRecoQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') + add_QC_finalization('emcBCQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json') add_QC_finalization('tpcStandardQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-standard-direct.json') add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-digits-task.json') diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 6ada53a15..744e6c3e9 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1241,6 +1241,11 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): needs=[EMCRECOtask['name']], readerCommand='o2-emcal-cell-reader-workflow --infile emccells.root', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') + if isActive('CTP'): + addQCPerTF(taskName='emcBCQC', + needs=[EMCRECOtask['name'], getDigiTaskName("CTP")], + readerCommand='o2-emcal-cell-reader-workflow --infile emccells.root | o2-ctp-digit-reader --inputfile ctpdigits.root --disable-mc', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/emc-bc-task.json') ### FT0 addQCPerTF(taskName='RecPointsQC', needs=[FT0RECOtask['name']], diff --git a/MC/config/QC/json/emc-bc-task.json b/MC/config/QC/json/emc-bc-task.json new file mode 100644 index 000000000..dd07cfa09 --- /dev/null +++ b/MC/config/QC/json/emc-bc-task.json @@ -0,0 +1,49 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "BCTaskEMCAL": { + "active": "true", + "taskName": "BCs", + "className": "o2::quality_control_modules::emcal::BCTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" + }, + "taskParameters": { + "AliasMB":"MINBIAS_TVX_L0,C0TVX,CMTVXTSC,C0TVXTSC,C0TVXTCE" + } + } + }, + "checks": {} + }, + "dataSamplingPolicies": [] + } \ No newline at end of file From 03ba9dd6cac8e42d664e1c57c8ab9af8e2a7fe4d Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 4 Dec 2023 11:27:56 +0100 Subject: [PATCH 0243/1239] Change what JDL can set for sec vtx and use of TPConly for V0s --- DATA/production/configurations/asyncReco/setenv_extra.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index b83c22fd5..cf908a5c7 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -429,10 +429,8 @@ fi if [[ $ALIEN_JDL_DISABLECASCADES == 1 ]]; then export ARGS_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=" --disable-cascade-finder " fi -# allow usage of TPC-only in svertexer (default: do not) -if [[ $ALIEN_JDL_USETPCONLYFORV0S == 1 ]]; then - export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=false" -else +# allow usage of TPC-only in svertexer (default: do use them, as in default in O2 and CCDB) +if [[ $ALIEN_JDL_DISABLETPCONLYFORV0S == 1 ]]; then export CONFIG_EXTRA_PROCESS_o2_secondary_vertexing_workflow+=";svertexer.mExcludeTPCtracks=true" fi From f7f3aca8e12f800593d79215c5e26df2d1e7aff9 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Thu, 30 Nov 2023 23:49:29 +0100 Subject: [PATCH 0244/1239] Align SOR and first orbit with CTPScalers * Take SOX and EOX from CTPScalers and this as the default time range for anchored MC * with --use-rct-info the old behaviour of taking the SOR and EOR info from RCT/Info/RunInformation is used --- MC/bin/o2dpg_sim_workflow_anchored.py | 91 +++++++++++++++++---------- 1 file changed, 58 insertions(+), 33 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index fed5ded96..236da81d5 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -187,7 +187,7 @@ def retrieve_sor_eor_fromGRPECS(ccdbreader, run_number, rct = None): print ("RunStart:", SOR) orbitFirst = int((1000*SOR - oreset[0])//LHCOrbitMUS) # calc done in microseconds - orbitLast = int((1000*EOR - oreset[0])//LHCOrbitMUS) + orbitLast = int((1000*EOR - oreset[0])//LHCOrbitMUS) print ("OrbitFirst", orbitFirst) # first orbit of this run print ("LastOrbit of run", orbitLast) @@ -213,35 +213,43 @@ def retrieve_GRPLHCIF(ccdbreader, timestamp): _, grplhcif = ccdbreader.fetch("GLO/Config/GRPLHCIF", "o2::parameters::GRPLHCIFData", timestamp = timestamp) return grplhcif -def retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, run_number, finaltime, trig_eff, NBunches, ColSystem): +def retrieve_CTPScalers(ccdbreader, run_number, timestamp=None): """ retrieves the CTP scalers object for a given timestamp and run_number and calculates the interation rate to be applied in Monte Carlo digitizers """ path = "CTP/Calib/Scalers/runNumber=" + str(run_number) - ts, ctpscaler = ccdbreader.fetch(path, "o2::ctp::CTPRunScalers", timestamp = timestamp) - if ctpscaler != None: + _, ctpscaler = ccdbreader.fetch(path, "o2::ctp::CTPRunScalers", timestamp = timestamp) + if ctpscaler is not None: ctpscaler.convertRawToO2() - # this is the default for pp - ctpclass = 0 # <---- we take the scaler for FT0 - ctptype = 0 - # this is the default for PbPb - if ColSystem == "PbPb": - ctpclass = 25 # <--- we take scalers for ZDC - ctptype = 7 - print("Fetching rate with class " + str(ctpclass) + " type " + str(ctptype)) - rate = ctpscaler.getRateGivenT(finaltime, ctpclass, ctptype) - #if ColSystem == "PbPb": - # rate.first = rate.first / 28. - # rate.second = rate.second / 28. - - print("Global rate " + str(rate.first) + " local rate " + str(rate.second)) - if rate.first >= 0: - # calculate true rate (input from Chiara Zampolli) using number of bunches - coll_bunches = NBunches - mu = - math.log(1. - rate.second / 11245 / coll_bunches) / trig_eff - finalRate = coll_bunches * mu * 11245 - return finalRate + return ctpscaler + return None + +def retrieve_MinBias_CTPScaler_Rate(ctpscaler, finaltime, trig_eff, NBunches, ColSystem): + """ + retrieves the CTP scalers object for a given timestamp and run_number + and calculates the interation rate to be applied in Monte Carlo digitizers + """ + # this is the default for pp + ctpclass = 0 # <---- we take the scaler for FT0 + ctptype = 0 + # this is the default for PbPb + if ColSystem == "PbPb": + ctpclass = 25 # <--- we take scalers for ZDC + ctptype = 7 + print("Fetching rate with class " + str(ctpclass) + " type " + str(ctptype)) + rate = ctpscaler.getRateGivenT(finaltime, ctpclass, ctptype) + #if ColSystem == "PbPb": + # rate.first = rate.first / 28. + # rate.second = rate.second / 28. + + print("Global rate " + str(rate.first) + " local rate " + str(rate.second)) + if rate.first >= 0: + # calculate true rate (input from Chiara Zampolli) using number of bunches + coll_bunches = NBunches + mu = - math.log(1. - rate.second / 11245 / coll_bunches) / trig_eff + finalRate = coll_bunches * mu * 11245 + return finalRate print (f"[ERROR]: Could not determine interaction rate; Some (external) default used") return None @@ -275,7 +283,7 @@ def determine_timestamp(sor, eor, splitinfo, cycle, ntf, HBF_per_timeframe = 256 maxcycles = maxtimeframesperjob // ntf print (f"We can do this amount of cycle iterations to achieve 100%: ", maxcycles) - + production_offset = int(thisjobID * maxcycles) + cycle timestamp_of_production = sor + production_offset * ntf * HBF_per_timeframe * LHCOrbitMUS / 1000 assert (timestamp_of_production >= sor) @@ -293,6 +301,7 @@ def main(): parser.add_argument("-tf", type=int, help="number of timeframes per job", default=1) parser.add_argument("--ccdb-IRate", type=bool, help="whether to try fetching IRate from CCDB/CTP", default=True) parser.add_argument("--trig-eff", type=float, dest="trig_eff", help="Trigger eff needed for IR", default=-1.0) + parser.add_argument("--use-rct-info", dest="use_rct_info", action="store_true", help=argparse.SUPPRESS) # Use SOR and EOR information from RCT instead of SOX and EOX from CTPScalers parser.add_argument('forward', nargs=argparse.REMAINDER) # forward args passed to actual workflow creation args = parser.parse_args() @@ -308,17 +317,33 @@ def main(): print ("No time info found") sys.exit(1) + ctp_scalers = retrieve_CTPScalers(ccdbreader, args.run_number) + if ctp_scalers is None: + print(f"ERROR: Cannot retrive scalers for run number {args.run_number}") + exit (1) + + first_orbit = ctp_scalers.getOrbitLimit().first + sor = ctp_scalers.getTimeLimit().first + eor = ctp_scalers.getTimeLimit().second + + if args.use_rct_info: + first_orbit = GLOparams["FirstOrbit"] + sor = GLOparams["SOR"] + eor = GLOparams["EOR"] + # determine timestamp, and production offset for the final # MC job to run - timestamp, prod_offset = determine_timestamp(GLOparams["SOR"], GLOparams["EOR"], [args.split_id - 1, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) + timestamp, prod_offset = determine_timestamp(sor, eor, [args.split_id - 1, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) + # this is anchored to - print ("Determined start-of-run to be: ", GLOparams["SOR"]) + print ("Determined start-of-run to be: ", sor) + print ("Determined end-of-run to be: ", eor) print ("Determined timestamp to be : ", timestamp) print ("Determined offset to be : ", prod_offset) - print ("Determined start of run to be : ", GLOparams["SOR"]) - currentorbit = GLOparams["FirstOrbit"] + prod_offset * GLOparams["OrbitsPerTF"] # orbit number at production start - currenttime = GLOparams["SOR"] + prod_offset * GLOparams["OrbitsPerTF"] * LHCOrbitMUS // 1000 # timestamp in milliseconds + currentorbit = first_orbit + prod_offset * GLOparams["OrbitsPerTF"] # orbit number at production start + currenttime = sor + prod_offset * GLOparams["OrbitsPerTF"] * LHCOrbitMUS // 1000 # timestamp in milliseconds + print ("Production put at time : " + str(currenttime)) # retrieve the GRPHCIF object @@ -361,7 +386,7 @@ def main(): else: effTrigger = 0.759 - rate = retrieve_MinBias_CTPScaler_Rate(ccdbreader, timestamp, args.run_number, currenttime/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) + rate = retrieve_MinBias_CTPScaler_Rate(ctp_scalers, currenttime/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) if rate != None: # if the rate calculation was successful we will use it, otherwise we fall back to some rate given as part @@ -371,10 +396,10 @@ def main(): # Use re.sub() to replace the pattern with an empty string forwardargs = re.sub(pattern, " ", forwardargs) forwardargs += ' -interactionRate ' + str(int(rate)) - + # we finally pass forward to the unanchored MC workflow creation # TODO: this needs to be done in a pythonic way clearly - forwardargs += " -tf " + str(args.tf) + " --sor " + str(GLOparams["SOR"]) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(GLOparams["FirstOrbit"]) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) + forwardargs += " -tf " + str(args.tf) + " --sor " + str(sor) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(first_orbit) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) print ("forward args ", forwardargs) cmd = "${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py " + forwardargs print ("Creating time-anchored workflow...") From 20473d5cfaac5c16feb162b8cbc5f1d24a38ff9c Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:30:04 +0100 Subject: [PATCH 0245/1239] TPC - Cleanup of qc files after QC review (#1347) * Update tpc.json * Update tpc.json --- DATA/production/qc-async/tpc.json | 17 ++++------------- DATA/production/qc-sync/tpc.json | 8 +------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index 10351d4ed..428814b7c 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -3,10 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "aaa", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" + "host": "ccdb-test.cern.ch:8080" }, "Activity": { "number": "42", @@ -23,7 +20,7 @@ "url": "" }, "conditionDB": { - "url": "alice-ccdb.cern.ch" + "url": "ccdb-test.cern.ch:8080" } }, "tasks": { @@ -34,8 +31,6 @@ "moduleName": "QcTPC", "detectorName": "TPC", "cycleDurationSeconds": "60", - "resetAfterCycles": "1", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "tpc-clusters" @@ -57,8 +52,6 @@ "moduleName": "QcTPC", "detectorName": "TPC", "cycleDurationSeconds": "60", - "resetAfterCycles": "1", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query" : "inputTracks:TPC/TRACKS/0" @@ -81,8 +74,6 @@ "moduleName": "QcTPC", "detectorName": "TPC", "cycleDurationSeconds": "60", - "resetAfterCycles": "1", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "tpc-tracks" @@ -109,7 +100,7 @@ { "id": "tpc-clusters", "active": "true", - "machines": ["localhost"], + "machines": [], "query" : "inputClusters:TPC/CLUSTERNATIVE", "outputs": "sampled-clusters:DS/CLUSTERNATIVE", "samplingConditions": [ @@ -130,7 +121,7 @@ { "condition": "random", "fraction": "0.1", - "seed": "1234" + "seed": "0" } ], "blocking": "false" diff --git a/DATA/production/qc-sync/tpc.json b/DATA/production/qc-sync/tpc.json index d11c1581a..5d0b22997 100644 --- a/DATA/production/qc-sync/tpc.json +++ b/DATA/production/qc-sync/tpc.json @@ -4,9 +4,6 @@ "database": { "implementation": "CCDB", "host": "ali-qcdb.cern.ch:8083", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable", "maxObjectSize" : "10000000" }, "Activity": { @@ -31,7 +28,6 @@ "detectorName": "TPC", "cycleDurationSeconds": "60", "resetAfterCycles": "1", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "random-cluster" @@ -74,7 +70,6 @@ "detectorName": "TPC", "cycleDurationSeconds": "60", "resetAfterCycles": "10", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "inputTracks:TPC/TRACKS/0" @@ -106,7 +101,6 @@ "detectorName": "TPC", "cycleDurationSeconds": "60", "resetAfterCycles": "10", - "maxNumberCycles": "-1", "dataSource": { "type": "dataSamplingPolicy", "name": "tpc-tracks" @@ -157,7 +151,7 @@ { "condition": "random", "fraction": "0.01", - "seed": "1234" + "seed": "0" } ], "blocking": "false" From e530167dfa5413563f34e83d368948137905eecb Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:30:25 +0100 Subject: [PATCH 0246/1239] TPC - cleanup specs in calibration workflows (#1348) * remove INFORMATION in spec * Remove INFORMATION for all calin specs --- DATA/production/calib/tpc-laser-filter.sh | 4 ++-- DATA/production/calib/tpc-laser.sh | 4 ++-- DATA/production/calib/tpc-pedestal.sh | 4 ++-- DATA/production/calib/tpc-pulser.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 677c14f90..43f7cf638 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -39,8 +39,8 @@ if [ $HOSTMEMSIZE != "0" ]; then GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" PROXY_OUTSPEC="A:TPC/LASERTRACKS;B:TPC/CEDIGITS;D:TPC/CLUSREFS" diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index 83d616895..f1de9d29b 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -39,8 +39,8 @@ if [ $HOSTMEMSIZE != "0" ]; then GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=450;TPCCalibPulser.LastTimeBin=550;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=2;TPCCalibPulser.XmaxQtot=502;TPCCalibPulser.MinimumQtot=8;TPCCalibPulser.MinimumQmax=6;TPCCalibPulser.XminT0=450;TPCCalibPulser.XmaxT0=550;TPCCalibPulser.NbinsT0=400;keyval.output_dir=/dev/null" diff --git a/DATA/production/calib/tpc-pedestal.sh b/DATA/production/calib/tpc-pedestal.sh index 80e2f003c..0261c3822 100755 --- a/DATA/production/calib/tpc-pedestal.sh +++ b/DATA/production/calib/tpc-pedestal.sh @@ -10,8 +10,8 @@ if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" CALIB_CONFIG="TPCCalibPedestal.LastTimeBin=12000;keyval.output_dir=/dev/null" diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index f7934e352..b3e9bde67 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -11,8 +11,8 @@ if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" CALIB_CONFIG="TPCCalibPulser.FirstTimeBin=80;TPCCalibPulser.LastTimeBin=260;TPCCalibPulser.NbinsQtot=250;TPCCalibPulser.XminQtot=10;TPCCalibPulser.XmaxQtot=510;TPCCalibPulser.NbinsWidth=100;TPCCalibPulser.XminWidth=0.3;TPCCalibPulser.XmaxWidth=0.7;TPCCalibPulser.MinimumQtot=30;TPCCalibPulser.MinimumQmax=25;TPCCalibPulser.XminT0=125;TPCCalibPulser.XmaxT0=145;TPCCalibPulser.NbinsT0=800;keyval.output_dir=/dev/null" From 0cbc119d42a20d88578140720a4bf203b15ec44c Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Mon, 4 Dec 2023 13:30:43 +0100 Subject: [PATCH 0247/1239] TPC Krypton - Add option to disable File writeout (#1338) * add option to disable writeout * Incrase pipline for raw-to-digit --- DATA/testing/detectors/TPC/tpc-krypton.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index cf7c83473..a25960b9d 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -24,7 +24,11 @@ ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/od QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" +WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR" +if [[ ${TPC_KRYPTON_NO_WRITEOUT:-} == 1 ]]; then + WRITER_TYPE="--writer-type none" +fi o2-dpl-raw-proxy $ARGS_ALL \ @@ -34,13 +38,11 @@ o2-dpl-raw-proxy $ARGS_ALL \ --input-spec "$CALIB_INSPEC" \ --configKeyValues "$ARGS_FILES" \ --remove-duplicates \ - --pipeline tpc-raw-to-digits-0:12 \ + --pipeline tpc-raw-to-digits-0:20 \ | o2-tpc-krypton-clusterer $ARGS_ALL \ + ${WRITER_TYPE} \ --lanes $NLANES \ --configKeyValues "$ARGS_FILES" \ --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ - --writer-type EPN \ - --meta-output-dir $EPN2EOS_METAFILES_DIR \ - --output-dir $CALIB_DIR \ | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host localhost \ | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} From 1dcf27a36369eb9ca75da7996c47faaee0d2596f Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Mon, 4 Dec 2023 14:13:25 +0100 Subject: [PATCH 0248/1239] TPC residual extraction with ITS-TPC tracks only --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index cf908a5c7..ec89015bb 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -352,7 +352,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then echo "Final setting for TPC scaling is:" echo $TPC_CORR_SCALING - + if [[ $PERIOD != @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]] ; then echo "Setting TPCCLUSTERTIMESHIFT to 0" TPCCLUSTERTIMESHIFT=0 @@ -509,7 +509,7 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 export CONFIG_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="scdcalib.maxTracksPerCalibSlot=35000000;scdcalib.minPtNoOuterPoint=0.2;scdcalib.maxQ2Pt=5;scdcalib.minITSNClsNoOuterPoint=6;scdcalib.minITSNCls=4;scdcalib.minTPCNClsNoOuterPoint=90" - export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --process-seeds" + export ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_scdcalib_interpolation_workflow --tracking-sources ITS-TPC" # ad-hoc settings for TPC residual extraction export ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator="$ARGS_EXTRA_PROCESS_o2_calibration_residual_aggregator --output-type trackParams,unbinnedResid" if [[ $ALIEN_JDL_DEBUGRESIDUALEXTRACTION == "1" ]]; then From 6f4eb9199b22b41aa1e778f877e887384cdc7b98 Mon Sep 17 00:00:00 2001 From: noferini <9963644+noferini@users.noreply.github.com> Date: Tue, 5 Dec 2023 09:40:16 +0100 Subject: [PATCH 0249/1239] fix ft0 ctp input for pp case --- MC/bin/o2dpg_sim_workflow_anchored.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 236da81d5..2feba79ce 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -232,7 +232,7 @@ def retrieve_MinBias_CTPScaler_Rate(ctpscaler, finaltime, trig_eff, NBunches, Co """ # this is the default for pp ctpclass = 0 # <---- we take the scaler for FT0 - ctptype = 0 + ctptype = 1 # this is the default for PbPb if ColSystem == "PbPb": ctpclass = 25 # <--- we take scalers for ZDC From 069ce4020f5187b802e9134a501495133c9ba991 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Tue, 5 Dec 2023 09:31:57 +0100 Subject: [PATCH 0250/1239] Fix ft0reco crash Recent disabling of FT0 calibrations in the reco workflow internally disables the CCDB fetcher. In result, the option --condition-not-before vanishes and the workflow crashes when the option is given on the command line. This is a generic problem with "dynamic" options. For now fixing, by taking away this option for FT0 reconstruction. --- MC/bin/o2dpg_sim_workflow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 744e6c3e9..2e1883730 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -964,7 +964,8 @@ def getDigiTaskName(det): workflow['stages'].append(ITSRECOtask) FT0RECOtask=createTask(name='ft0reco_'+str(tf), needs=[getDigiTaskName("FT0")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') - FT0RECOtask['cmd'] = '${O2_ROOT}/bin/o2-ft0-reco-workflow --disable-time-offset-calib --disable-slewing-calib ' + getDPL_global_options() + putConfigValues() + # note: when calibrations (or CCDB objects) are reenabled, we need to say ccdbbackend=True + FT0RECOtask['cmd'] = '${O2_ROOT}/bin/o2-ft0-reco-workflow --disable-time-offset-calib --disable-slewing-calib ' + getDPL_global_options(ccdbbackend=False) + putConfigValues() workflow['stages'].append(FT0RECOtask) ITSTPCMATCHtask=createTask(name='itstpcMatch_'+str(tf), needs=[TPCRECOtask['name'], ITSRECOtask['name'], FT0RECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='8000', relative_cpu=3/8) From 6c6417bc0b28577e515b5b788fd98e0e97768021 Mon Sep 17 00:00:00 2001 From: swenzel Date: Wed, 6 Dec 2023 10:07:42 +0100 Subject: [PATCH 0251/1239] Improvements in failure log processing * ... in particular less connections to AliEn --- GRID/utils/extractErroredLogFiles.sh | 17 +++++++++--- GRID/utils/extractFailureMessage.sh | 3 ++- GRID/utils/fetch_output_onfailure.sh | 39 ++++++++++++++++++---------- 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/GRID/utils/extractErroredLogFiles.sh b/GRID/utils/extractErroredLogFiles.sh index 39ce80e79..a8d579585 100755 --- a/GRID/utils/extractErroredLogFiles.sh +++ b/GRID/utils/extractErroredLogFiles.sh @@ -1,6 +1,15 @@ # We determine the O2DPG task that failed (as listed in stdout) and extract the relevant log automatically # Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs +alias mytar=tar +if [[ $(uname) == "Darwin" ]]; then + echo "Running on macOS. This needs gnu-tar" + alias mytar=gtar +fi + +# show aliases +alias + errored_tasks="" find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*//' | tr ":" " " | while IFS= read -r line; do stdoutpath=$(echo "$line" | awk '{print $1}') # Extracting the first column @@ -17,12 +26,12 @@ find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*/ # extract the general log archive unzip -n log_archive.zip # extract specific task log from debug archive - tar -xvzf debug_log_archive.tgz --wildcards "*${logfile}.log" + mytar -xvzf debug_log_archive.tgz --wildcards "*${logfile}.log" if [[ ${logfile} == *"qedsim"* || ${logfile} == *"sgnsim"* || ${logfile} == *"bkgsim"* ]]; then # simulation has few more files to inspect - tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*serverlog*" - tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*workerlog*" - tar -xvzf debug_log_archive.tgz --wildcards "*${tf}*mergerlog*" + mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*serverlog*" + mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*workerlog*" + mytar -xvzf debug_log_archive.tgz --wildcards "*${tf}*mergerlog*" fi cd $OLDPWD done diff --git a/GRID/utils/extractFailureMessage.sh b/GRID/utils/extractFailureMessage.sh index ea78ff2e7..548666e9a 100755 --- a/GRID/utils/extractFailureMessage.sh +++ b/GRID/utils/extractFailureMessage.sh @@ -23,6 +23,7 @@ pattern="-e \"\<[Ee]xception\" \ -e \"terminate called without an active\" \ -e \"\]\[FATAL\]\" \ -e \"\*\*\* Program crashed\" \ + -e \"\*\*\* Fatal Exception\" \ -e \"\*\*\* Error in\"" # <--- LIBC fatal error messages CMD="grep -m 1 -B2 -A10 ${pattern} $filetocheck" @@ -32,4 +33,4 @@ if [ "$?" -eq 0 ]; then echo "==============================ERROR in ${filetocheck} ==============================" echo "" echo "${T}" -fi \ No newline at end of file +fi diff --git a/GRID/utils/fetch_output_onfailure.sh b/GRID/utils/fetch_output_onfailure.sh index 076ef996e..a9f991109 100755 --- a/GRID/utils/fetch_output_onfailure.sh +++ b/GRID/utils/fetch_output_onfailure.sh @@ -8,27 +8,40 @@ if [ ! "${MY_JOBID}" ]; then exit 1 fi +# these are all subjobids SUBJOBIDS=($(alien.py ps --trace ${MY_JOBID} | awk '/Subjob submitted/' | sed 's/.*submitted: //' | tr '\n' ' ')) +# these are the one that failed +FAILEDSUBJOBIDS=($(alien.py ps -a -m ${MY_JOBID} -f ERROR_ALL | awk '/EE/{print $2}' | tr '\n' ' ')) + CurrDir=${PWD} OutputDir=/tmp/AlienLogs_${MY_JOBID} -# -job_number=${#SUBJOBIDS[@]} - - -# We fetch the error files locally +job_number=${#FAILEDSUBJOBIDS[@]} +echo "Found ${job_number} failed job ids" +echo "Registering output for retrieval" +RecycleBase="" +# First pass to do register output for ((i = 0; i < job_number; i++)); do - jobid=${SUBJOBIDS[i]} - STATUS=$(alien.py ps -j ${jobid} | awk '//{print $4}') - echo "Status of ${jobid} is $STATUS" - if [ ${STATUS} == "EE" ]; then - # nothing + jobid=${FAILEDSUBJOBIDS[i]} + if [ ! "${RecycleBase}" ]; then RecycleOutputDir=$(alien.py ps --trace ${jobid} | awk '/Going to uploadOutputFiles/' | sed 's/.*outputDir=//' | sed 's/)//') - alien.py registerOutput ${jobid} - echo "Recycle out is ${RecycleOutputDir}" - alien.py cp ${RecycleOutputDir}/'*' file:${OutputDir}/${jobid} + # /alice/cern.ch/user/a/aliprod/recycle/alien-job-2974093751 + RecycleBase=${RecycleOutputDir%-${jobid}} # Removes the ${jobid} and yields the recycle base path fi + $(alien.py registerOutput ${jobid}) 2> /dev/null +done + +# wait a bit to allow propagation of "registerOutput" +sleep 1 + +echo "Downloading output" +# Second pass to copy files +for ((i = 0; i < job_number; i++)); do + jobid=${FAILEDSUBJOBIDS[i]} + RecycleOutputDir="${RecycleBase}-${jobid}" + alien.py cp ${RecycleOutputDir}/'*archive*' file:${OutputDir}/${jobid} + [ -f ${OutputDir}/${jobid}/log_archive.zip ] && unzip -q -o ${OutputDir}/${jobid}/log_archive.zip -d ${OutputDir}/${jobid} done echo " ... Going to automatic extraction of log files ... " From 50f7e88062bdc50b1825d28c7615f8968a1e7e5c Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 5 Dec 2023 12:25:01 +0100 Subject: [PATCH 0252/1239] Adding possibility to scale with derivatives --- DATA/production/configurations/asyncReco/setenv_extra.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index ec89015bb..98d915153 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -326,6 +326,9 @@ elif [[ $ALIGNLEVEL == 1 ]]; then elif [[ $INST_IR_FOR_TPC == "IDCCCDB" ]]; then echo "TPC correction with IDC from CCDB will be used" export TPC_CORR_SCALING+=" --lumi-type 2 " + if [[ $ALIEN_JDL_USEDERIVATIVESFORSCALING == "1" ]]; then + export TPC_CORR_SCALING+=" --corrmap-lumi-mode 1 " + fi else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" return 1 From 8b4e86ea16708fb244a0530e7370365005627f95 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Wed, 6 Dec 2023 20:18:00 +0100 Subject: [PATCH 0253/1239] TPC Laser calibration - ARGS_ALL_CONFIG fix (#1368) * remove INFORMATION in spec * Remove INFORMATION for all calin specs * Add fix for ALL_ALL_CONFIG loading * remove ARGS_ALL_CONFIG from laser workflows --- DATA/production/calib/tpc-laser-filter.sh | 1 - DATA/production/calib/tpc-laser.sh | 2 -- 2 files changed, 3 deletions(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index 43f7cf638..a13267b8a 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -10,7 +10,6 @@ FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" FILEWORKDIR2="/home/epn/odc/files/" -ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" diff --git a/DATA/production/calib/tpc-laser.sh b/DATA/production/calib/tpc-laser.sh index f1de9d29b..859fd362b 100755 --- a/DATA/production/calib/tpc-laser.sh +++ b/DATA/production/calib/tpc-laser.sh @@ -10,8 +10,6 @@ FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" FILEWORKDIR2="/home/epn/odc/files/" -ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" - if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi From 2335e79c530e874f332a23c4fc90df5c2984ea11 Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Wed, 6 Dec 2023 13:11:21 +0100 Subject: [PATCH 0254/1239] Updated tpc config to include DCA downsampling and lower pT cut --- DATA/production/qc-async/tpc.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index 428814b7c..a715c29e7 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -81,7 +81,9 @@ "taskParameters": { "cutAbsEta": "1.", "cutMinNCluster": "60", - "cutMindEdxTot": "20." + "cutMindEdxTot": "20.", + "samplingFractionDCAr": "0.1", + "cutPtForDCAr": "1.5" }, "grpGeomRequest" : { "geomRequest": "None", From 23ad22328de1fe76a00e78039a8ce60876f50e18 Mon Sep 17 00:00:00 2001 From: JianLIUhep Date: Mon, 28 Aug 2023 09:49:45 +0200 Subject: [PATCH 0255/1239] Added ITS V0 reconstruction in async workflow --- DATA/production/qc-async/its.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/qc-async/its.json b/DATA/production/qc-async/its.json index 82a5c0345..ff8c0408b 100644 --- a/DATA/production/qc-async/its.json +++ b/DATA/production/qc-async/its.json @@ -79,7 +79,8 @@ "doTTree": "0", "nBCbins" : "103", "dicttimestamp" : 0, - "doNorm": "1" + "doNorm": "1", + "InvMasses" : "1" } } }, From 2c337e3f2d79a90dc2ac9304baf99f3bd138a472 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 7 Dec 2023 09:38:04 +0100 Subject: [PATCH 0256/1239] Update workflow example shell script and clean up default ARGS_ALL_CONFIG --- DATA/common/getCommonArgs.sh | 2 +- DATA/testing/examples/example-workflow.sh | 24 +++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/DATA/common/getCommonArgs.sh b/DATA/common/getCommonArgs.sh index dd4b5d551..f1458789d 100755 --- a/DATA/common/getCommonArgs.sh +++ b/DATA/common/getCommonArgs.sh @@ -10,7 +10,7 @@ if [[ -z $SEVERITY || -z $NUMAID || -z $SHMSIZE || -z $FILEWORKDIR || -z $EPNSYN fi ARGS_ALL="--session ${OVERRIDE_SESSION:-default} --severity $SEVERITY --shm-segment-id ${O2JOBSHMID:-$NUMAID} --shm-segment-size $SHMSIZE ${ARGS_ALL_EXTRA:-} --early-forward-policy noraw" -ARGS_ALL_CONFIG="NameConf.mDirGeom=$FILEWORKDIR;NameConf.mDirGRP=$FILEWORKDIRRUN;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;${ALL_EXTRA_CONFIG:-}" +ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;${ALL_EXTRA_CONFIG:-}" if [[ $EPNSYNCMODE == 1 ]]; then ARGS_ALL+=" --infologger-severity $INFOLOGGER_SEVERITY" ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock --resources-monitoring 15" diff --git a/DATA/testing/examples/example-workflow.sh b/DATA/testing/examples/example-workflow.sh index deefb230a..ea3c5a16f 100755 --- a/DATA/testing/examples/example-workflow.sh +++ b/DATA/testing/examples/example-workflow.sh @@ -1,17 +1,21 @@ #!/usr/bin/env bash +# Make common arguments and helper functions such as add_W available source common/setenv.sh - -SEVERITY=warning source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh +# Define input data required by DPL (in this case all RAWDATA from TPC) PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" --inject-missing-data \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$PROXY_INSPEC" \ - --remove-duplicates \ - --configKeyValues "$ARGS_ALL_CONFIG;TPCDigitDump.LastTimeBin=1000;" \ - | o2-dpl-run $ARGS_ALL $GLOBALDPLOPT --dds ${WORKFLOWMODE_FILE} +# Start with an empty workflow +WORKFLOW= +# Add required workflows via add_W helper (usage: add_W [BINARY] [COMMAND_LINE_OPTIONS] [CONFIG_KEY_VALUES] [Add ARGS_ALL_CONFIG, optional, default = 1]) +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --readout-proxy \"--channel-config \\\"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\\\"\"" "" 0 +add_W o2-tpc-raw-to-digits-workflow "--input-spec \"$PROXY_INSPEC\" --remove-duplicates" "TPCDigitDump.LastTimeBin=1000" + +# Finally add the o2-dpl-run workflow manually, allow for either printing the workflow or creating a topology (default) +WORKFLOW+="o2-dpl-run $GLOBALDPLOPT $ARGS_ALL" +[[ $WORKFLOWMODE != "print" ]] && WORKFLOW+=" --${WORKFLOWMODE} ${WORKFLOWMODE_FILE:-}" +[[ $WORKFLOWMODE == "print" || "${PRINT_WORKFLOW:-}" == "1" ]] && echo "#Workflow command:\n\n${WORKFLOW}\n" | sed -e "s/\\\\n/\n/g" -e"s/| */| \\\\\n/g" | eval cat $( [[ $WORKFLOWMODE == "dds" ]] && echo '1>&2') +if [[ $WORKFLOWMODE != "print" ]]; then eval $WORKFLOW; else true; fi From 34057be0ca5fa359216002ecb5271c1291a36dfe Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 7 Dec 2023 09:51:19 +0100 Subject: [PATCH 0257/1239] Update detector workflows --- DATA/testing/detectors/FT0/ft0-digits-qc-ctf.sh | 11 ++++++----- DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh | 10 ++++++---- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 7 ++++--- DATA/testing/detectors/TPC/tpc-krypton.sh | 7 ++++--- DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh | 7 ++++--- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/DATA/testing/detectors/FT0/ft0-digits-qc-ctf.sh b/DATA/testing/detectors/FT0/ft0-digits-qc-ctf.sh index 8495cb385..b1869604e 100755 --- a/DATA/testing/detectors/FT0/ft0-digits-qc-ctf.sh +++ b/DATA/testing/detectors/FT0/ft0-digits-qc-ctf.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash +# Make common arguments and helper functions such as add_W available source common/setenv.sh +source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh -SEVERITY=WARNING -ARGS_ALL="--session default --severity $SEVERITY --shm-segment-id $NUMAID --shm-segment-size $SHMSIZE" -ARGS_ALL+=" --infologger-severity $SEVERITY" if [ -z $CTF_DIR ]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store dictionary files -#ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock" -ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" + CTF_DICT="--ctf-dict $FILEWORKDIR/ctf_dictionary.root" NTHREADS=2 # Output directory for the CTF, to write to the current dir., remove `--output-dir $CTFOUT` from o2-ctf-writer-workflow or set to `CTFOUT=\"""\"` @@ -18,6 +17,8 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FT0/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" +# TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow +# as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" \ | o2-ft0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline ft0-datareader-dpl:$NTHREADS \ | o2-ft0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ diff --git a/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh b/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh index 72f226e31..05c170095 100755 --- a/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh +++ b/DATA/testing/detectors/FV0/fv0-digits-qc-ctf.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash +# Make common arguments and helper functions such as add_W available source common/setenv.sh +source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh + -SEVERITY=WARNING -ARGS_ALL="--session default --severity $SEVERITY --shm-segment-id $NUMAID --shm-segment-size $SHMSIZE" -ARGS_ALL+=" --infologger-severity $SEVERITY" if [ -z $CTF_DIR ]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store dictionary files #ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock" -ARGS_ALL_CONFIG="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" CTF_DICT="--ctf-dict $FILEWORKDIR/ctf_dictionary.root" NTHREADS=2 # Output directory for the CTF, to write to the current dir., remove `--output-dir $CTFOUT` from o2-ctf-writer-workflow or set to `CTFOUT=\"""\"` @@ -18,6 +18,8 @@ MYDIR="$(dirname $(readlink -f $0))" PROXY_INSPEC="x:FV0/RAWDATA;eos:***/INFORMATION;dd:FLP/DISTSUBTIMEFRAME/0" IN_CHANNEL="--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1'" +# TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow +# as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh o2-dpl-raw-proxy ${ARGS_ALL} --readout-proxy "${IN_CHANNEL}" --dataspec "${PROXY_INSPEC}" --inject-missing-data \ | o2-fv0-flp-dpl-workflow --disable-root-output ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" --pipeline fv0-datareader-dpl:$NTHREADS \ | o2-fv0-entropy-encoder-workflow ${ARGS_ALL} --configKeyValues "$ARGS_ALL_CONFIG;" ${CTF_DICT} \ diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index f400ca4a5..bbdfa743b 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash source common/setenv.sh - source common/getCommonArgs.sh +source common/gen_topo_helper_functions.sh export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder @@ -12,11 +12,12 @@ CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" NLANES=36 SESSION="default" -ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" +ARGS_FILES="keyval.output_dir=/dev/null" HOST=localhost QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" - +# TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow +# as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index a25960b9d..d7dc474eb 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -4,7 +4,7 @@ source common/setenv.sh source common/getCommonArgs.sh -source common/gen_topo_helper_functions.sh +source common/gen_topo_helper_functions.sh export GLOBAL_SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder @@ -20,7 +20,7 @@ CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" NLANES=1 SESSION="default" -ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" +ARGS_FILES="keyval.output_dir=/dev/null" QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" @@ -30,7 +30,8 @@ if [[ ${TPC_KRYPTON_NO_WRITEOUT:-} == 1 ]]; then WRITER_TYPE="--writer-type none" fi - +# TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow +# as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ diff --git a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh index 13526e29c..cfdaa0ffc 100755 --- a/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh +++ b/DATA/testing/detectors/TPC/tpc-laser-raw-filter.sh @@ -4,7 +4,7 @@ source common/setenv.sh source common/getCommonArgs.sh -source common/gen_topo_helper_functions.sh +source common/gen_topo_helper_functions.sh export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder @@ -18,13 +18,14 @@ CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" NLANES=36 SESSION="default" PIPEADD="0" -ARGS_FILES="NameConf.mDirGRP=/home/epn/odc/files/;NameConf.mDirGeom=/home/epn/odc/files/;keyval.output_dir=/dev/null" +ARGS_FILES="keyval.output_dir=/dev/null" HOST=localhost QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" - +# TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow +# as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh o2-dpl-raw-proxy $ARGS_ALL \ --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ From e94e7f0cd140ca865397d396c8ba8fc3ac19c6e6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 10:18:51 +0100 Subject: [PATCH 0258/1239] dpl-workflow: tune 4gpu pbpb multiplicities --- DATA/production/workflow-multiplicities.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 9e55359af..1704f0a07 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -16,6 +16,8 @@ SOURCE_GUARD_MULTIPLICITIES=1 : ${ITSTRK_THREADS:=1} : ${ITSTPC_THREADS:=1} +: ${TOFMATCH_THREADS:=1} + : ${HIGH_RATE_PP:=0} # FIXME: multithreading in the itsmft reconstruction does not work on macOS. @@ -98,24 +100,25 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_TPCITS=12 N_ITSTRK=12 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then - [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=20 - [[ -z ${SHMSIZE:-} ]] && SHMSIZE=128000000000 # SHM_LIMIT 3/4 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=30 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000 # SHM_LIMIT 3/4 [[ -z ${TIMEFRAME_SHM_LIMIT:-} ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3)) NGPURECOTHREADS=8 - NTRDTRKTHREADS=4 - ITSTRK_THREADS=6 + NTRDTRKTHREADS=8 + ITSTRK_THREADS=20 ITSTPC_THREADS=3 - SVERTEX_THREADS=40 + SVERTEX_THREADS=20 + TOFMATCH_THREADS=2 N_SECVTX=1 NGPUS=4 N_TPCTRK=4 # time in s: pvtx 16, tof 30, trd 82 itstpc 53 its 200 mfttr 30 tpcent 23 hmp-clus 40 (25.11.22) N_TPCENTDEC=$(math_max $((5 * $NGPUS / 4)) 1) - N_ITSTRK=$(math_max $((10 * $NGPUS / 4)) 1) - N_TPCITS=$(math_max $((7 * $NGPUS / 4)) 1) + N_ITSTRK=$(math_max $((4 * $NGPUS / 4)) 1) + N_TPCITS=$(math_max $((5 * $NGPUS / 4)) 1) N_MFTTRK=$(math_max $((3 * $NGPUS / 4)) 1) N_TRDTRK=$(math_max $((9 * $NGPUS / 4)) 1) - N_TOFMATCH=$(math_max $((11 * $NGPUS / 4)) 1) + N_TOFMATCH=$(math_max $((6 * $NGPUS / 4)) 1) N_HMPCLUS=$(math_max $((3 * $NGPUS / 4)) 1) N_AODPROD=2 N_MCHCL=9 From 17d70dd5cd10c48d92627a551c3c8b9fd3d382f0 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 11:02:40 +0100 Subject: [PATCH 0259/1239] Fix whitespace, convert tabs to spaces --- .../configurations/asyncReco/async_pass.sh | 196 +++++++++--------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 2999b3e43..366cfbb1e 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -122,7 +122,7 @@ export timeUsed=0 ###if [[ $MODE == "remote" ]]; then # run specific archive if [[ ! -f runInput_$RUNNUMBER.tgz ]]; then - echo "No runInput_$RUNNUMBER.tgz, let's hope we don't need it" + echo "No runInput_$RUNNUMBER.tgz, let's hope we don't need it" else tar -xzvf runInput_$RUNNUMBER.tgz fi @@ -324,8 +324,8 @@ if [[ -n $ALIEN_JDL_PACKAGES ]]; then # if we have this env variable, it means t break else if [[ "$((SUBJOBIDX%KEEPRATIO))" -eq "0" ]]; then - keep=1 - break + keep=1 + break fi fi done < tmp.tmp @@ -360,32 +360,32 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ "0$ASYNC_PASS_NO_OPTIMIZED_DEFAULTS" != "01" ]]; then if [[ "ALIEN_JDL_USEFULLNUMADOMAIN" == 0 ]]; then if [[ $keep -eq 0 ]]; then - if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then - export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs) - else - # forcing multiplicities to be 1 - export MULTIPLICITY_PROCESS_tof_matcher=1 - export MULTIPLICITY_PROCESS_mch_cluster_finder=1 - export MULTIPLICITY_PROCESS_tpc_entropy_decoder=1 - export MULTIPLICITY_PROCESS_itstpc_track_matcher=1 - export MULTIPLICITY_PROCESS_its_tracker=1 - export OMP_NUM_THREADS=4 - fi - export TIMEFRAME_RATE_LIMIT=8 + if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then + export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs) + else + # forcing multiplicities to be 1 + export MULTIPLICITY_PROCESS_tof_matcher=1 + export MULTIPLICITY_PROCESS_mch_cluster_finder=1 + export MULTIPLICITY_PROCESS_tpc_entropy_decoder=1 + export MULTIPLICITY_PROCESS_itstpc_track_matcher=1 + export MULTIPLICITY_PROCESS_its_tracker=1 + export OMP_NUM_THREADS=4 + fi + export TIMEFRAME_RATE_LIMIT=8 else - export TIMEFRAME_RATE_LIMIT=4 + export TIMEFRAME_RATE_LIMIT=4 fi export SHMSIZE=30000000000 else export DPL_SMOOTH_RATE_LIMITING=1 if [[ $BEAMTYPE == "pp" ]]; then - export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) - export TIMEFRAME_RATE_LIMIT=45 - export SHMSIZE=100000000000 + export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) + export TIMEFRAME_RATE_LIMIT=45 + export SHMSIZE=100000000000 else # PbPb - export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb) - export TIMEFRAME_RATE_LIMIT=20 - export SHMSIZE=128000000000 + export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb) + export TIMEFRAME_RATE_LIMIT=20 + export SHMSIZE=128000000000 fi fi fi @@ -398,18 +398,18 @@ else if [[ "$ALIEN_JDL_EPNFULLNUMACPUONLY" != 1 ]]; then export TIMEFRAME_RATE_LIMIT=3 if [[ $BEAMTYPE == "pp" ]]; then - if (( $(echo "$RUN_IR > 800000" | bc -l) )); then - export TIMEFRAME_RATE_LIMIT=1 - elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then - export TIMEFRAME_RATE_LIMIT=6 - fi - export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) - export SHMSIZE=16000000000 + if (( $(echo "$RUN_IR > 800000" | bc -l) )); then + export TIMEFRAME_RATE_LIMIT=1 + elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then + export TIMEFRAME_RATE_LIMIT=6 + fi + export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) + export SHMSIZE=16000000000 else # PbPb - export TIMEFRAME_RATE_LIMIT=2 - export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu - export SHMSIZE=16000000000 - export SVERTEX_THREADS=5 + export TIMEFRAME_RATE_LIMIT=2 + export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu + export SHMSIZE=16000000000 + export SVERTEX_THREADS=5 fi else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs @@ -488,20 +488,20 @@ else echo "Time spent in running the workflow, Step 1 = $delta s" echo "exitcode = $exitcode" if [[ $exitcode -ne 0 ]]; then - echo "exit code from Step 1 of processing is " $exitcode > validation_error.message - echo "exit code from Step 1 of processing is " $exitcode - exit $exitcode + echo "exit code from Step 1 of processing is " $exitcode > validation_error.message + echo "exit code from Step 1 of processing is " $exitcode + exit $exitcode fi mv latest.log latest_reco_1.log if [[ -f performanceMetrics.json ]]; then - mv performanceMetrics.json performanceMetrics_1.json + mv performanceMetrics.json performanceMetrics_1.json fi $STATSCRIPT latest_reco_1.log reco_1 exitcode=$? if [[ $exitcode -ne 0 ]]; then - echo "exit code from processing is " $exitcode > validation_error.message - echo "exit code from processing is " $exitcode - exit $exitcode + echo "exit code from processing is " $exitcode > validation_error.message + echo "exit code from processing is " $exitcode + exit $exitcode fi fi fi @@ -526,43 +526,43 @@ else echo "Time spent in running the workflow, Step 2 = $delta s" echo "exitcode = $exitcode" if [[ $exitcode -ne 0 ]]; then - echo "exit code from Step 2 of processing is " $exitcode > validation_error.message - echo "exit code from Step 2 of processing is " $exitcode - exit $exitcode + echo "exit code from Step 2 of processing is " $exitcode > validation_error.message + echo "exit code from Step 2 of processing is " $exitcode + exit $exitcode fi mv latest.log latest_reco_2.log if [[ -f performanceMetrics.json ]]; then - mv performanceMetrics.json performanceMetrics_2.json + mv performanceMetrics.json performanceMetrics_2.json fi $STATSCRIPT latest_reco_2.log reco_2 exitcode=$? if [[ $exitcode -ne 0 ]]; then - echo "exit code from processing is " $exitcode > validation_error.message - echo "exit code from processing is " $exitcode - exit $exitcode + echo "exit code from processing is " $exitcode > validation_error.message + echo "exit code from processing is " $exitcode + exit $exitcode fi # let's compare to previous step if [[ -f latest_reco_1.log ]]; then - nCTFsFilesInspected_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/\(^[0-9]*\)_.*/\1/'` - nCTFsFilesOK_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_\([0-9]*\)_.*/\1/'` - nCTFsFilesFailed_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_[0-9]*_\([0-9]*\)_.*/\1/'` - nCTFsProcessed_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_[0-9]*_[0-9]*_\([0-9]*\).*/\1/'` - nCTFsFilesInspected_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/\(^[0-9]*\)_.*/\1/'` - nCTFsFilesOK_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_\([0-9]*\)_.*/\1/'` - nCTFsFilesFailed_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/^[0-9]*_[0-9]*_\([0-9]*\)_.*/\1/'` - nCTFsProcessed_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/^[0-9]*_[0-9]*_[0-9]*_\([0-9]*\).*/\1/'` - if [[ $nCTFsFilesInspected_step1 != $nCTFsFilesInspected_step2 ]] || [[ $nCTFsFilesFailed_step1 != $nCTFsFilesFailed_step2 ]] || [[ $nCTFsFilesOK_step1 != $nCTFsFilesOK_step2 ]] || [[ $nCTFsProcessed_step1 != $nCTFsProcessed_step2 ]]; then - echo "Inconsistency between step 1 and step 2 in terms of number of CTFs (files or single CTFs) found:" - echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" - echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" - echo "nCTFsFilesFailed_step1 = $nCTFsFilesFailed_step1, nCTFsFilesFailed_step2 = $nCTFsFilesFailed_step2" - echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" - echo "Inconsistency between step 1 and step 2 in terms of number of CTFs (files or single CTFs) found:" > validation_error.message - echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" > validation_error.message - echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" > validation_error.message - echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" > validation_error.message - exit 1000 - fi + nCTFsFilesInspected_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/\(^[0-9]*\)_.*/\1/'` + nCTFsFilesOK_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_\([0-9]*\)_.*/\1/'` + nCTFsFilesFailed_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_[0-9]*_\([0-9]*\)_.*/\1/'` + nCTFsProcessed_step1=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_[0-9]*_[0-9]*_\([0-9]*\).*/\1/'` + nCTFsFilesInspected_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/\(^[0-9]*\)_.*/\1/'` + nCTFsFilesOK_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_1.stat | sed 's/^[0-9]*_\([0-9]*\)_.*/\1/'` + nCTFsFilesFailed_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/^[0-9]*_[0-9]*_\([0-9]*\)_.*/\1/'` + nCTFsProcessed_step2=`ls [0-9]*_[0-9]*_[0-9]*_[0-9]*_[0-9]*_reco_2.stat | sed 's/^[0-9]*_[0-9]*_[0-9]*_\([0-9]*\).*/\1/'` + if [[ $nCTFsFilesInspected_step1 != $nCTFsFilesInspected_step2 ]] || [[ $nCTFsFilesFailed_step1 != $nCTFsFilesFailed_step2 ]] || [[ $nCTFsFilesOK_step1 != $nCTFsFilesOK_step2 ]] || [[ $nCTFsProcessed_step1 != $nCTFsProcessed_step2 ]]; then + echo "Inconsistency between step 1 and step 2 in terms of number of CTFs (files or single CTFs) found:" + echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" + echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" + echo "nCTFsFilesFailed_step1 = $nCTFsFilesFailed_step1, nCTFsFilesFailed_step2 = $nCTFsFilesFailed_step2" + echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" + echo "Inconsistency between step 1 and step 2 in terms of number of CTFs (files or single CTFs) found:" > validation_error.message + echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" > validation_error.message + echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" > validation_error.message + echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" > validation_error.message + exit 1000 + fi fi fi fi @@ -589,13 +589,13 @@ else echo "Time spent in running the workflow, Step 3 = $delta s" echo "exitcode = $exitcode" if [[ $exitcode -ne 0 ]]; then - echo "exit code from Step 3 of processing is " $exitcode > validation_error.message - echo "exit code from Step 3 of processing is " $exitcode - exit $exitcode + echo "exit code from Step 3 of processing is " $exitcode > validation_error.message + echo "exit code from Step 3 of processing is " $exitcode + exit $exitcode fi mv latest.log latest_reco_3.log if [[ -f performanceMetrics.json ]]; then - mv performanceMetrics.json performanceMetrics_3.json + mv performanceMetrics.json performanceMetrics_3.json fi fi fi @@ -655,9 +655,9 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then echo "Time spent in merging last AOD files, to reach a good size for that too = $delta s" echo "exitcode = $exitcode" if [[ $exitcode -ne 0 ]]; then - echo "exit code from aod-merger for latest file is " $exitcode > validation_error.message - echo "exit code from aod-merger for latest file is " $exitcode - exit $exitcode + echo "exit code from aod-merger for latest file is " $exitcode > validation_error.message + echo "exit code from aod-merger for latest file is " $exitcode + exit $exitcode fi MERGED_SIZE=`wc -c AO2D.root | awk '{print $1}'` echo "Size of merged file: $MERGED_SIZE" @@ -702,8 +702,8 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then timeEndCheck=`date +%s` timeUsedCheck=$(( $timeUsedCheck+$timeEndCheck-$timeStartCheck )) if [[ $exitcode -ne 0 ]]; then - echo "exit code from AO2D check is " $exitcode > validation_error.message - echo "exit code from AO2D check is " $exitcode + echo "exit code from AO2D check is " $exitcode > validation_error.message + echo "exit code from AO2D check is " $exitcode fi fi cd - @@ -719,8 +719,8 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then i=0 while IFS= read -r line; do while [[ $CURRENT_POOL_SIZE -ge $MAX_POOL_SIZE ]]; do - CURRENT_POOL_SIZE=`jobs -r | wc -l` - sleep 1 + CURRENT_POOL_SIZE=`jobs -r | wc -l` + sleep 1 done run_AOD_merging $line & arr[$i]=$! @@ -733,13 +733,13 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then wait ${arr[$i]} exitcode=$? if [[ $exitcode -ne 0 ]]; then - echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode > validation_error.message - echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode - echo "As a consequence, we will keep the AO2Ds with unmerged DFs for ${aods[$i]}" - mergedok[$((10#${aods[$i]}))]=0 + echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode > validation_error.message + echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode + echo "As a consequence, we will keep the AO2Ds with unmerged DFs for ${aods[$i]}" + mergedok[$((10#${aods[$i]}))]=0 else - echo "Merging of DFs inside the AO2D in ${aods[$i]} worked correctly" - mergedok[$((10#${aods[$i]}))]=1 + echo "Merging of DFs inside the AO2D in ${aods[$i]} worked correctly" + mergedok[$((10#${aods[$i]}))]=1 fi done timeEndMerge=`date +%s` @@ -751,10 +751,10 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then AOD_DIR=`dirname $AOD_FILE | sed -e 's|./||'` echo "Inspecting $AOD_DIR:" if [[ ${mergedok[$((10#$AOD_DIR))]} == 0 ]]; then - echo "Merging for $AOD_DIR DID NOT work, we will do nothing for this file" - continue + echo "Merging for $AOD_DIR DID NOT work, we will do nothing for this file" + continue else - echo "Merging for $AOD_DIR DID work, let's continue" + echo "Merging for $AOD_DIR DID work, let's continue" fi cd $AOD_DIR # now checking them @@ -765,12 +765,12 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then timeEndCheckMergedAOD=`date +%s` timeUsedCheckMergedAOD=$(( $timeUsedCheckMergedAOD+$timeEndCheckMergedAOD-$timeStartCheckMergedAOD )) if [[ $exitcode -ne 0 ]]; then - echo "exit code from AO2D with merged DFs check is " $exitcode > validation_error.message - echo "exit code from AO2D with merged DFs check is " $exitcode - echo "We will keep the AO2Ds with unmerged DFs" + echo "exit code from AO2D with merged DFs check is " $exitcode > validation_error.message + echo "exit code from AO2D with merged DFs check is " $exitcode + echo "We will keep the AO2Ds with unmerged DFs" else - echo "All ok, replacing initial AO2D.root file with the one with merged DFs" - mv AO2D_merged.root AO2D.root + echo "All ok, replacing initial AO2D.root file with the one with merged DFs" + mv AO2D_merged.root AO2D.root fi cd .. done @@ -791,17 +791,17 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then timeUsedAnalysisQC=$(( $timeUsedAnalysisQC+$timeEndAnalysisQC-$timeStartAnalysisQC )) echo "exitcode = $exitcode" if [[ $exitcode -ne 0 ]]; then - echo "exit code from Analysis QC is " $exitcode > validation_error.message - echo "exit code from Analysis QC is " $exitcode - exit $exitcode + echo "exit code from Analysis QC is " $exitcode > validation_error.message + echo "exit code from Analysis QC is " $exitcode + exit $exitcode fi if [[ -f "Analysis/MergedAnalyses/AnalysisResults.root" ]]; then - mv Analysis/MergedAnalyses/AnalysisResults.root . + mv Analysis/MergedAnalyses/AnalysisResults.root . else - echo "No Analysis/MergedAnalyses/AnalysisResults.root found! check analysis QC" + echo "No Analysis/MergedAnalyses/AnalysisResults.root found! check analysis QC" fi if ls Analysis/*/*.log 1> /dev/null 2>&1; then - mv Analysis/*/*.log . + mv Analysis/*/*.log . fi cd .. done From 83a9daf2625cbb20d8203ad2b06f7f1323bfc4c7 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 11:08:26 +0100 Subject: [PATCH 0260/1239] async_pass: synchronize settings with workflow-multiplicities --- .../configurations/asyncReco/async_pass.sh | 18 +++++++++--------- DATA/production/workflow-multiplicities.sh | 6 +++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 366cfbb1e..6ffa8b7a7 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -362,6 +362,7 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ $keep -eq 0 ]]; then if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs) + export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 else # forcing multiplicities to be 1 export MULTIPLICITY_PROCESS_tof_matcher=1 @@ -370,22 +371,21 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then export MULTIPLICITY_PROCESS_itstpc_track_matcher=1 export MULTIPLICITY_PROCESS_its_tracker=1 export OMP_NUM_THREADS=4 + export TIMEFRAME_RATE_LIMIT=8 + export SHMSIZE=30000000000 fi - export TIMEFRAME_RATE_LIMIT=8 else export TIMEFRAME_RATE_LIMIT=4 + export SHMSIZE=30000000000 fi - export SHMSIZE=30000000000 else export DPL_SMOOTH_RATE_LIMITING=1 if [[ $BEAMTYPE == "pp" ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) - export TIMEFRAME_RATE_LIMIT=45 - export SHMSIZE=100000000000 + export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 else # PbPb export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb) - export TIMEFRAME_RATE_LIMIT=20 - export SHMSIZE=128000000000 + export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 fi fi fi @@ -396,12 +396,13 @@ else # if [[ "0$ASYNC_PASS_NO_OPTIMIZED_DEFAULTS" != "01" ]]; then if [[ "$ALIEN_JDL_EPNFULLNUMACPUONLY" != 1 ]]; then - export TIMEFRAME_RATE_LIMIT=3 if [[ $BEAMTYPE == "pp" ]]; then if (( $(echo "$RUN_IR > 800000" | bc -l) )); then export TIMEFRAME_RATE_LIMIT=1 elif (( $(echo "$RUN_IR < 50000" | bc -l) )); then export TIMEFRAME_RATE_LIMIT=6 + else + export TIMEFRAME_RATE_LIMIT=3 fi export OPTIMIZED_PARALLEL_ASYNC=pp_8cpu # sets the multiplicities to optimized defaults for this configuration (grid) export SHMSIZE=16000000000 @@ -413,8 +414,7 @@ else fi else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs - export TIMEFRAME_RATE_LIMIT=32 - SHMSIZE=90000000000 + export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 fi fi fi diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 1704f0a07..495491184 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -40,6 +40,10 @@ N_F_CTF=$MULTIPLICITY_FACTOR_CTFENCODERS N_TPCTRK=$NGPUS if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then # Tuned multiplicities for async processing + if [[ ${OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT:-} == 1 ]]; then + [[ ! -z ${TIMEFRAME_RATE_LIMIT:-} ]] && unset TIMEFRAME_RATE_LIMIT + [[ ! -z ${SHMSIZE:-} ]] && unset SHMSIZE + fi if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_8cpu" ]]; then [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=3 NGPURECOTHREADS=5 @@ -66,7 +70,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_TPCENTDEC=3 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_1gpu" ]]; then [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8 - [[ -z ${SHMSIZE:-} ]] && SHMSIZE=20000000000 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=30000000000 N_TOFMATCH=2 N_MCHCL=3 N_TPCENTDEC=2 From 0ce5ceb26bf4a2064a127dad957b4fdb5d6fe92d Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 11:17:47 +0100 Subject: [PATCH 0261/1239] workflow-multiplicities: use 100% larger SHM size and tf rate limit on MI100 nodes (50% for sync) --- DATA/production/workflow-multiplicities.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 495491184..cca4d41ec 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -144,6 +144,11 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then echo "Invalid optimized setting '$OPTIMIZED_PARALLEL_ASYNC'" 1>&2 exit 1 fi + if [[ ${OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT:-} == 1 && ${EPN_NODE_MI100:-} == 1 ]]; then + TIMEFRAME_RATE_LIMIT=$(($TIMEFRAME_RATE_LIMIT * 2)) + SHMSIZE=$(($SHMSIZE * 2)) + EPN_GLOBAL_SCALING=2 + fi elif [[ $EPNPIPELINES != 0 ]]; then NTRDTRKTHREADS=2 ITSTRK_THREADS=2 From a29f4ac7cce0e4c7976ead792aaa25ae4922a1e9 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 12:09:32 +0100 Subject: [PATCH 0262/1239] More cleanup and debug reporting --- DATA/production/configurations/asyncReco/async_pass.sh | 3 +-- DATA/production/workflow-multiplicities.sh | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 6ffa8b7a7..941657c6d 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -379,7 +379,6 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then export SHMSIZE=30000000000 fi else - export DPL_SMOOTH_RATE_LIMITING=1 if [[ $BEAMTYPE == "pp" ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 @@ -419,7 +418,7 @@ else fi fi -echo "[INFO (async_pass.sh)] envvars were set to TFDELAYSECONDS ${TFDELAYSECONDS} TIMEFRAME_RATE_LIMIT ${TIMEFRAME_RATE_LIMIT}" +echo "[INFO (async_pass.sh)] envvars were set to TFDELAYSECONDS ${TFDELAYSECONDS} TIMEFRAME_RATE_LIMIT ${TIMEFRAME_RATE_LIMIT} OPTIMIZED_PARALLEL_ASYNC ${OPTIMIZED_PARALLEL_ASYNC:-NONE}" [[ -z $NTIMEFRAMES ]] && export NTIMEFRAMES=-1 diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index cca4d41ec..fb0580566 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -46,6 +46,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then fi if [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_8cpu" ]]; then [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=3 + [[ -z ${SHMSIZE:-} ]] && SHMSIZE=16000000000 NGPURECOTHREADS=5 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "pp_16cpu" ]]; then [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=8 @@ -103,6 +104,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_TPCENTDEC=6 N_TPCITS=12 N_ITSTRK=12 + export DPL_SMOOTH_RATE_LIMITING=1 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=30 [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000 # SHM_LIMIT 3/4 @@ -130,6 +132,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_MCHTRK=7 N_PRIMVTXMATCH=2 N_PRIMVTX=3 + export DPL_SMOOTH_RATE_LIMITING=1 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_64cpu" ]]; then NGPURECOTHREADS=6 NTRDTRKTHREADS=2 From fd619fe62c5c4f622492992a367da6945f1d5186 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 12:16:43 +0100 Subject: [PATCH 0263/1239] Temporary workaround, to be reverted once O2 PR 12412 is in all async tags --- DATA/production/configurations/asyncReco/async_pass.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 941657c6d..5cf53e89c 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -363,6 +363,8 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 + export TIMEFRAME_RATE_LIMIT=8 # WORKAROUND: Needed until O2 PR 12412 is in all async tags + export SHMSIZE=30000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags else # forcing multiplicities to be 1 export MULTIPLICITY_PROCESS_tof_matcher=1 @@ -382,9 +384,13 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ $BEAMTYPE == "pp" ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 + export TIMEFRAME_RATE_LIMIT=45 # WORKAROUND: Needed until O2 PR 12412 is in all async tags + export SHMSIZE=100000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags else # PbPb export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 + export TIMEFRAME_RATE_LIMIT=30 # WORKAROUND: Needed until O2 PR 12412 is in all async tags + export SHMSIZE=100000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags fi fi fi @@ -414,6 +420,8 @@ else else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 + export TIMEFRAME_RATE_LIMIT=32 # WORKAROUND: Needed until O2 PR 12412 is in all async tags + export SHMSIZE=90000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags fi fi fi From f6a2fcea894b90b5f99c4759bc4cb7d8c6db5951 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Mon, 20 Nov 2023 10:50:10 +0100 Subject: [PATCH 0264/1239] Towards more dynamic/flexible usage of RelVal * in short: more modularisation * now also the extraction step leaves a JSON config with object names, path to flat ROOT file and meta info such as assigned label ==> easier to re-do/re-plot ==> this JSON file can be seen as a configuration which allows to have flat ROOT files "anywhere" and only this configuration is needed to run RelVal * overlay plotting * is now its own step and disentangled from metrics' calculations * moved to PyROOT for better control and access or replotting * all changes transparent w.r.t. user API * add shell script as example and fast way of running RelVal for QC * short update of README --- RelVal/PlotOverlays.C | 339 ------------------ RelVal/README.md | 8 + RelVal/o2dpg_overlay_plots.py | 122 +++---- RelVal/o2dpg_release_validation.py | 191 ++++++---- RelVal/run/run_data_rel_val.sh | 26 ++ RelVal/{ => utils}/ExtractAndFlatten.C | 77 ++-- RelVal/{ => utils}/ReleaseValidation.C | 24 +- RelVal/{ => utils}/ReleaseValidationMetrics.C | 4 +- .../o2dpg_release_validation_plot.py | 27 +- .../o2dpg_release_validation_plot_root.py | 313 ++++++++++++++++ .../o2dpg_release_validation_utils.py | 20 +- .../o2dpg_release_validation_variables.py | 0 12 files changed, 618 insertions(+), 533 deletions(-) delete mode 100644 RelVal/PlotOverlays.C create mode 100755 RelVal/run/run_data_rel_val.sh rename RelVal/{ => utils}/ExtractAndFlatten.C (71%) rename RelVal/{ => utils}/ReleaseValidation.C (91%) rename RelVal/{ => utils}/ReleaseValidationMetrics.C (97%) rename RelVal/{ => utils}/o2dpg_release_validation_plot.py (82%) create mode 100755 RelVal/utils/o2dpg_release_validation_plot_root.py rename RelVal/{ => utils}/o2dpg_release_validation_utils.py (97%) rename RelVal/{ => utils}/o2dpg_release_validation_variables.py (100%) diff --git a/RelVal/PlotOverlays.C b/RelVal/PlotOverlays.C deleted file mode 100644 index 27a2c141f..000000000 --- a/RelVal/PlotOverlays.C +++ /dev/null @@ -1,339 +0,0 @@ -#include -#include -#include - - - -void findRangeNotEmpty1D(TH1* h, double& min, double& max) -{ - auto axis = h->GetXaxis(); - min = axis->GetBinLowEdge(1); - max = axis->GetBinUpEdge(axis->GetNbins()); - for (int i = 1; i <= axis->GetNbins(); i++) { - if (h->GetBinContent(i) != 0) { - min = axis->GetBinLowEdge(i); - break; - } - } - for (int i = axis->GetNbins(); i >= 1 ; i--) { - if (h->GetBinContent(i) != 0) { - max = axis->GetBinUpEdge(i); - break; - } - } -} - - -TH1* makeFrameFromHistograms(TPad& pad, std::vector const& histograms, bool& shouldBeLog) -{ - // make a frame to fit all histograms - // propose log10 scale in case some integrals differ by more than 2 orders of magnitude - auto integralRef = histograms[0]->Integral(); - shouldBeLog = false; - auto minY = histograms[0]->GetMinimum(0); - double maxY = histograms[0]->GetMaximum(); - - double minX; - double maxX; - findRangeNotEmpty1D(histograms[0], minX, maxX); - - // find minima and maxima - for (int i = 1; i < histograms.size(); i++) { - minY = std::min(histograms[i]->GetMinimum(0), minY); - maxY = std::max(histograms[i]->GetMaximum(), maxY); - - double minXNext; - double maxXNext; - findRangeNotEmpty1D(histograms[i], minXNext, maxXNext); - minX = std::min(minX, minXNext); - maxX = std::max(maxX, maxXNext); - - auto integral = histograms[i]->Integral(); - if ((integralRef > 0 && integral / integralRef > 100) || (integral > 0 && integralRef / integral > 100)) { - // decide whether to do a log plot - shouldBeLog = true; - } - } - - // finalise the y-axis limits - if (shouldBeLog) { - auto margin = std::log10(maxY / minY); - minY = minY / std::pow(10, margin * 0.1); - maxY = maxY * std::pow(10, margin * 0.3);; - } else { - auto margin = 0.1 * (maxY - minY); - maxY += 3 * margin; - minY -= std::max(0., margin); - } - - if (histograms[0]->GetXaxis()->IsAlphanumeric()) { - auto alphanumericFrame = (TH1*)histograms[0]->Clone(); - alphanumericFrame->Reset("ICEMS"); - return alphanumericFrame; - } - - return pad.DrawFrame(minX, minY, maxX, maxY); -} - -// overlay 1D histograms -void overlay1D(std::vector hVec, std::vector labelVec, TLegend* additionalLegend, std::string const& outputDir) -{ - TCanvas c("overlay", "", 800, 800); - c.cd(); - - TPad nominalPad("nominalPad", "nominalPad", 0, 0.3, 1., 1.); - nominalPad.SetBottomMargin(0); - TPad ratioPad("ratioPad", "ratioPad", 0, 0.05, 1. ,0.32); - ratioPad.SetTopMargin(0); - ratioPad.SetBottomMargin(0.2); - - nominalPad.Draw(); - ratioPad.Draw(); - - const int colors[7] = {kRed + 2, kBlue - 4, kGreen + 3, kMagenta + 1, kCyan + 2, kOrange + 5, kYellow - 6}; - const int linestyles[6] = {1, 10, 2, 9, 8, 7}; - - TLegend legendOverlay(0.65, 0.7, 0.9, 0.9); - legendOverlay.SetFillStyle(0); - legendOverlay.SetBorderSize(0); - - bool logY{}; - nominalPad.cd(); - auto frame = makeFrameFromHistograms(nominalPad, hVec, logY); - frame->SetTitle(hVec[0]->GetTitle()); - auto yAxis = frame->GetYaxis(); - yAxis->ChangeLabel(1, -1, -1, -1, -1, -1, " "); - yAxis->SetTitleFont(43); - yAxis->SetTitleSize(20); - yAxis->SetLabelFont(43); - yAxis->SetLabelSize(20); - yAxis->SetTitle(hVec[0]->GetYaxis()->GetTitle()); - auto xAxis = frame->GetXaxis(); - xAxis->SetLabelFont(43); - xAxis->SetLabelSize(0); - - - std::vector ratios; - - std::string emptyText; - for (int i = 0; i < hVec.size(); i++) { - auto& h = hVec[i]; - - h->SetStats(0); - h->SetLineStyle(linestyles[i % 6]); - h->SetLineWidth(1); - h->SetLineColor(colors[i % 7]); - - if (i > 0) { - // no ratio for the first histogram (which would simply be 1) - TH1* hRatio = (TH1*)h->Clone(); - hRatio->SetDirectory(0); - hRatio->Divide(h, hVec[0], 1.0, 1.0, "B"); // error option? - ratios.push_back(hRatio); - } - - legendOverlay.AddEntry(h, labelVec[i].c_str()); - - h->Draw("same E hist"); - if (h->GetEntries() == 0) { - emptyText += labelVec[i] + ", "; - } - } - - if (logY) { - nominalPad.SetLogy(); - } - - if (additionalLegend) { - additionalLegend->SetBorderSize(0); - additionalLegend->SetFillStyle(0); - // To reposition the legend we need to: Draw, Update, set new coordinates, Modified - additionalLegend->Draw("same"); - nominalPad.Update(); - additionalLegend->SetX1NDC(0.15); - additionalLegend->SetY1NDC(0.7); - additionalLegend->SetX2NDC(0.4); - additionalLegend->SetY2NDC(0.9); - nominalPad.Modified(); - } - legendOverlay.Draw("same"); - - if (!emptyText.empty()) { - emptyText.pop_back(); - emptyText.pop_back(); - emptyText = std::string("EMPTY: ") + emptyText; - TText *t1 = new TText(0.2, 0.5, emptyText.c_str()); - t1->SetNDC(); - t1->Draw(); - } - - ratioPad.cd(); - frame = makeFrameFromHistograms(ratioPad, ratios, logY); - yAxis = frame->GetYaxis(); - yAxis->SetTitleFont(43); - yAxis->SetTitleSize(20); - yAxis->SetLabelFont(43); - yAxis->SetLabelSize(20); - yAxis->SetTitle("ratio"); - - xAxis = frame->GetXaxis(); - xAxis->SetTitleFont(43); - xAxis->SetTitleSize(20); - xAxis->SetLabelFont(43); - xAxis->SetLabelSize(20); - xAxis->SetTitle(hVec[0]->GetXaxis()->GetTitle()); - - for (int i = 0; i < ratios.size(); i++) { - auto& h = ratios[i]; - h->Draw("same"); - } - - if (logY) { - ratioPad.SetLogy(); - } - - auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; - c.SaveAs(savePath.c_str()); - c.Close(); - - for (auto& r : ratios) { - delete r; - } -} - -// overlay 2D histograms -void overlay2D(std::vector hVec1, std::vector labelVec, TLegend* legend, std::string const& outputDir) -{ - std::vector hVec; - for (auto h : hVec1){ - hVec.push_back(dynamic_cast(h)); - } - int nHistos = hVec.size(); - - TCanvas c("overlay", "", 2400, 800 * (nHistos-1)); - c.Divide(3, nHistos - 1); - c.cd(1); - hVec[0]->SetTitle(hVec[0]->GetTitle() + TString("(" + labelVec[0] + ")")); - hVec[0]->SetStats(0); - hVec[0]->Draw("colz"); - - if (hVec[0]->GetEntries() == 0) { - TText *t1 = new TText(0.5, 0.5, "EMPTY"); - t1->SetNDC(); - t1->Draw(); - } - - std::vector ratios; - - for (int i = 1; i < nHistos; i++){ - auto hDiv = (TH2*)hVec[i]->Clone(Form("%s_ratio", hVec[i]->GetName())); - hDiv->SetDirectory(0); - ratios.push_back(hDiv); - hDiv->SetTitle(hVec[i]->GetTitle() + TString("(" + labelVec[i] + "/"+labelVec[0]+")")); - hDiv->SetStats(0); - hDiv->Divide(hVec[0]); - hVec[i]->SetTitle(hVec[i]->GetTitle() + TString("(" + labelVec[i] + ")")); - hVec[i]->SetStats(0); - - c.cd(i * 3 - 1); - hVec[i]->Draw("colz"); - if (hVec[i]->GetEntries() == 0) { - TText *t1 = new TText(0.5, 0.5, "EMPTY"); - t1->SetNDC(); - t1->Draw(); - } - - c.cd(i * 3); - hDiv->Draw("colz"); - } - - if (legend){ - c.cd(3); - legend->SetTextSize(0.03); - legend->SetTextFont(62); - legend->Draw("same"); - gPad->Update(); - legend->SetX1NDC(0.4); - legend->SetY1NDC(0.7); - legend->SetX2NDC(0.89); - legend->SetY2NDC(0.89); - gPad->Modified(); - } - - auto savePath = outputDir + "/" + hVec[0]->GetName() + ".png"; - c.SaveAs(savePath.c_str()); - c.Close(); - - for (auto& r : ratios) { - delete r; - } -} - -// entry point for overlay plots from ReleaseValidation.C -void PlotOverlayAndRatio(std::vector hVec, std::vector labelVec, std::string outputDir = "overlayPlots", TLegend* legendMetrics = nullptr) -{ - if (!std::filesystem::exists(outputDir)) { - std::filesystem::create_directory(outputDir); - } - - bool is2D = false; - for (auto h : hVec){ - if (dynamic_cast(h)){ - std::cerr << "Cannot yet overlay 3D histograms\nSkipping " << h->GetName() << "\n"; - return; - } - if (dynamic_cast(h)){ - is2D = true; - } - } - - if (is2D){ - overlay2D(hVec, labelVec, legendMetrics, outputDir); - } - else { - overlay1D(hVec, labelVec, legendMetrics, outputDir); - } -} - -// entry point for plotting only overlays -void PlotOverlays(initializer_list fileNames_list, initializer_list labelVec_list, std::string outputDir = "overlayPlots"){ - std::vector fileNames(fileNames_list); - std::vector labelVec(labelVec_list); - - if (fileNames.size() > labelVec.size()){ - for (int i=labelVec.size();i files; - for (auto& fileName : fileNames){ - TFile* thisFile = TFile::Open(fileName.c_str(), "READ"); - files.push_back(thisFile); - } - - TIter next(files[0]->GetListOfKeys()); - TKey* key{}; - while ((key = static_cast(next()))) { - std::vector hVec; - hVec.push_back(static_cast(key->ReadObj())); - auto oname = key->GetName(); - bool foundAll = true; - for (int i=1; i(files[i]->Get(oname)); - if (!hNew) { - // That could still happen in case we compare either comletely different file by accident or something has been changed/added/removed - foundAll = false; - std::cerr << "ERROR: Histogram " << oname << " not found in file " << fileNames[i].c_str() << "\n"; - } - hVec.push_back(hNew); - } - if (foundAll){ - PlotOverlayAndRatio(hVec, labelVec, outputDir); - } - else { - std::cerr << "ERROR: Histogram " << oname << " not found in all files\n"; - return; - } - } -} diff --git a/RelVal/README.md b/RelVal/README.md index e50a897fc..98ac4af0e 100644 --- a/RelVal/README.md +++ b/RelVal/README.md @@ -127,6 +127,8 @@ ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${MC_PRODUCTION}/QC/ ## Run for QC This is a simple guide to run RelVal for QC. +Here is also a [working example](run/run_data_rel_val.sh). + ### If you are interested in all QC plots To have everything and to use this as a starting point for deeper inspections, first run ```bash @@ -149,6 +151,12 @@ ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i QC_file_1.root -j QC ``` From here on, you can use the `inspect` command as usual. But there will never be detectors other than ITS and CPV. +### Troubleshooting + +If there are unexpected segmentation faults or similar, most likely the `QualityControl` software is not properly linked against `O2`. Most likely, the reason is that `QC` was not rebuild against the loaded `O2` version. +The easiest solution would be to load either `QualityControl` or meta packages such as `O2sim`. +Loading like `O2/latest,QualityControl/latest` can cause problems depending on how the single packages were build. + ## Expert section ### Adding a new metric diff --git a/RelVal/o2dpg_overlay_plots.py b/RelVal/o2dpg_overlay_plots.py index 8f9bae311..9f9a420b7 100755 --- a/RelVal/o2dpg_overlay_plots.py +++ b/RelVal/o2dpg_overlay_plots.py @@ -2,10 +2,9 @@ import sys import argparse -from os import environ, makedirs, remove -from os.path import join, exists, abspath -from subprocess import Popen, PIPE, STDOUT -from shlex import split +import importlib.util +from os import environ +from os.path import join # make sure O2DPG + O2 is loaded O2DPG_ROOT=environ.get('O2DPG_ROOT') @@ -14,91 +13,54 @@ print('ERROR: This needs O2DPG loaded') sys.exit(1) -ROOT_MACRO_EXTRACT=join(O2DPG_ROOT, "RelVal", "ExtractAndFlatten.C") -ROOT_MACRO_OVERLAYS=join(O2DPG_ROOT, "RelVal", "PlotOverlays.C") - -FLATTENED_FILE_NAME = "newfile" - -def run_macro(cmd, log_file, output_dir): - p = Popen(split(cmd), cwd=output_dir, stdout=PIPE, stderr=STDOUT, universal_newlines=True) - log_file = open(log_file, 'a') - for line in p.stdout: - log_file.write(line) - p.wait() - log_file.close() - -def call_extract_and_flatten(inputs, output_dir): - output_dir = abspath(output_dir) - if not exists(output_dir): - makedirs(output_dir) - log_file_extract = join(abspath(output_dir), "extract_and_flatten.log") - counter = 0 - firstfile = join(output_dir,FLATTENED_FILE_NAME+str(1)+".root") - for f in inputs: - counter += 1 - print(f"Extraction of objects from {f}") - f = abspath(f) - newfile = join(output_dir,FLATTENED_FILE_NAME+str(counter)+".root") - if exists(newfile): - remove(newfile) - cmd = "" - if counter==1: - cmd = f"\\(\\\"{f}\\\",\\\"{newfile}\\\",\\\"\\\",\\\"\\\"\\)" - else: - cmd = f"\\(\\\"{f}\\\",\\\"{newfile}\\\",\\\"{firstfile}\\\",\\\"\\\"\\)" - cmd = f"root -l -b -q {ROOT_MACRO_EXTRACT}{cmd}" - run_macro(cmd, log_file_extract, output_dir) - return 0 +spec = importlib.util.spec_from_file_location("o2dpg_release_validation", join(O2DPG_ROOT, "RelVal", 'o2dpg_release_validation.py')) +o2dpg_release_validation = importlib.util.module_from_spec(spec) +spec.loader.exec_module(o2dpg_release_validation) +sys.modules["o2dpg_release_validation"] = o2dpg_release_validation +from o2dpg_release_validation import only_extract_impl -def call_plot_overlays(inputs, output_dir, labels, no_extract): - nInput = len(inputs) - output_dir = abspath(output_dir) - if not exists(output_dir): - makedirs(output_dir) - cmd = f"\\({{" - for i in range(nInput): - if not no_extract: - f = join(output_dir,FLATTENED_FILE_NAME+str(i+1)+".root") - else: - f = abspath(inputs[i]) - cmd = cmd + f"\\\"{f}\\\"," - cmd = cmd[:-1] - cmd = cmd + f"}},{{" - if labels: - for l in labels: - cmd = cmd + f"\\\"{l}\\\"," - cmd = cmd[:-1] - cmd = cmd + f"}},\\\"{output_dir}\\\"\\)" - cmd = f"root -l -b -q {ROOT_MACRO_OVERLAYS}{cmd}" - log_file_extract = join(output_dir, "plot_overlays.log") - print("Make the plots") - run_macro(cmd, log_file_extract, output_dir) - return 0 +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_plot.py')) +o2dpg_release_validation_plot = importlib.util.module_from_spec(spec) +spec.loader.exec_module(o2dpg_release_validation_plot) +sys.modules["o2dpg_release_validation_plot"] = o2dpg_release_validation_plot +from o2dpg_release_validation_plot import plot_overlays_no_rel_val + + +def run(args): + + if not args.labels: + args.labels = [f"label_{i}" for i, _ in enumerate(args.input)] + + if len(args.labels) != len(args.input): + print("ERROR: Number of input files and labels is different, must be the same") + return 1 + + out_configs = [] + ref_file = None + for i, (input_file, label) in enumerate(zip(args.inputs, args.labels)): + + _, config = only_extract_impl(input_file, args.output, label, prefix=i, reference_extracted=ref_file) + if not config: + print(f"ERROR: Problem with input file {input_file}, cannot extract") + return 1 + + if not ref_file: + ref_file = config["path"] + + out_configs.append(config) + + plot_overlays_no_rel_val(out_configs, args.output) -def clean_up(nInput, output_dir): - for i in range(nInput): - f = join(output_dir,FLATTENED_FILE_NAME+str(i+1)+".root") - remove(f) - print(f"Remove {f}") return 0 + def main(): parser = argparse.ArgumentParser() parser.add_argument("-i", "--input", nargs="*", help="list of ROOT files", required=True) parser.add_argument("-o", "--output", help="output directory", default="overlayPlots") parser.add_argument("-l", "--labels", nargs="*", help="plot labels") - parser.add_argument("--clean", help="delete newfile.root files after macro has finished", action="store_true") - parser.add_argument("--no-extract", dest="no_extract", help="no extraction but immediately expect histograms present for comparison", action="store_true") - args = parser.parse_args() - if not args.no_extract: - call_extract_and_flatten(args.input, args.output) - call_plot_overlays(args.input, args.output, args.labels, args.no_extract) - if args.clean: - if not args.no_extract: - clean_up(len(args.input), args.output) - else: - print("Nothing to clean up.") - return 0 + return run(parser.parse_args()) + if __name__ == "__main__": sys.exit(main()) diff --git a/RelVal/o2dpg_release_validation.py b/RelVal/o2dpg_release_validation.py index 2f1bd375e..c776bd624 100755 --- a/RelVal/o2dpg_release_validation.py +++ b/RelVal/o2dpg_release_validation.py @@ -14,6 +14,7 @@ from os import environ, makedirs, remove, rename from os.path import join, abspath, exists, dirname, basename, isfile from shutil import copy, rmtree +import json # make sure O2DPG + O2 is loaded O2DPG_ROOT=environ.get('O2DPG_ROOT') @@ -24,28 +25,28 @@ O2DPG_ROOT = environ.get("O2DPG_ROOT") -spec = importlib.util.spec_from_file_location("o2dpg_release_validation_variables", join(O2DPG_ROOT, "RelVal", 'o2dpg_release_validation_variables.py')) +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_variables", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_variables.py')) o2dpg_release_validation_variables = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_variables) sys.modules["o2dpg_release_validation_variables"] = o2dpg_release_validation_variables import o2dpg_release_validation_variables as variables -spec = importlib.util.spec_from_file_location("o2dpg_release_validation_utils", join(O2DPG_ROOT, "RelVal", 'o2dpg_release_validation_utils.py')) +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_utils", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_utils.py')) o2dpg_release_validation_utils = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_utils) sys.modules["o2dpg_release_validation_utils"] = o2dpg_release_validation_utils from o2dpg_release_validation_utils import * -spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot", join(O2DPG_ROOT, "RelVal", 'o2dpg_release_validation_plot.py')) +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_plot.py')) o2dpg_release_validation_plot = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_plot) sys.modules["o2dpg_release_validation_plot"] = o2dpg_release_validation_plot -from o2dpg_release_validation_plot import plot_pie_charts, plot_summary_grid, plot_compare_summaries +from o2dpg_release_validation_plot import plot_pie_charts, plot_summary_grid, plot_compare_summaries, plot_overlays -ROOT_MACRO_EXTRACT=join(O2DPG_ROOT, "RelVal", "ExtractAndFlatten.C") -ROOT_MACRO_RELVAL=join(O2DPG_ROOT, "RelVal", "ReleaseValidation.C") -ROOT_MACRO_METRICS=join(O2DPG_ROOT, "RelVal", "ReleaseValidationMetrics.C") +ROOT_MACRO_EXTRACT=join(O2DPG_ROOT, "RelVal", "utils", "ExtractAndFlatten.C") +ROOT_MACRO_RELVAL=join(O2DPG_ROOT, "RelVal", "utils", "ReleaseValidation.C") +ROOT_MACRO_METRICS=join(O2DPG_ROOT, "RelVal", "utils", "ReleaseValidationMetrics.C") from ROOT import gROOT @@ -114,7 +115,8 @@ def metrics_from_root(): current_metric = None return 0 -def extract(input_filenames, target_filename, include_file_directories="", add_if_exists=False, reference_extracted=""): + +def extract(input_filenames, target_filename, include_file_directories=None, add_if_exists=False, reference_extracted=None, json_extracted=None): """ Wrap the extraction of objects to be compared @@ -134,9 +136,31 @@ def extract(input_filenames, target_filename, include_file_directories="", add_i bool True in case of success, False otherwise """ + def get_files_from_list(list_filename): + """ + Quick helper + + Extract filenames from what is listed in a given file + """ + collect_files = [] + with open(list_filename, "r") as f: + for line in f: + line = line.strip() + if not line: + continue + collect_files.append(line) + return collect_files include_file_directories = ",".join(include_file_directories) if include_file_directories else "" + # flat ROOT files to extract to and read from during RelVal; make absolute paths so we don't confuse ourselves when running e.g. ROOT macros in different directories + + if len(input_filenames) == 1 and input_filenames[0][0] == "@": + input_filenames = get_files_from_list(input_filenames[0][1:]) + if not files1: + print(f"ERROR: Apparently {input_filenames[0][1:]} contains no files to be extracted.") + return None + if exists(target_filename) and not add_if_exists: # this file will otherwise be updated if it exists remove(target_filename) @@ -145,13 +169,17 @@ def extract(input_filenames, target_filename, include_file_directories="", add_i cwd = dirname(target_filename) target_filename = basename(target_filename) log_file_name = join(cwd, f"{target_filename}_extract_and_flatten.log") + if not reference_extracted: + reference_extracted = "" + if not json_extracted: + json_extracted = "" print("Extraction of files") for f in input_filenames: f = abspath(f) print(f" {f}") - cmd = f"\\(\\\"{f}\\\",\\\"{target_filename}\\\",\\\"{reference_extracted}\\\",\\\"{include_file_directories}\\\"\\)" + cmd = f"\\(\\\"{f}\\\",\\\"{target_filename}\\\",\\\"{reference_extracted}\\\",\\\"{include_file_directories}\\\",\\\"{json_extracted}\\\"\\)" cmd = f"root -l -b -q {ROOT_MACRO_EXTRACT}{cmd}" ret = run_macro(cmd, log_file_name, cwd) if ret != 0: @@ -160,8 +188,56 @@ def extract(input_filenames, target_filename, include_file_directories="", add_i return True +def get_extract_json_info(json_path): + + if not exists(json_path): + return None + + with open(json_path, "r") as f: + try: + return json.load(f) + except (json.decoder.JSONDecodeError, UnicodeDecodeError): + pass + return None + + +def only_extract_impl(files, output, label, include_directories=None, add_if_exists=False, prefix=None, reference_extracted=None): + + if len(files) == 1: + d = get_extract_json_info(files[0]) + if d is not None: + return files[0], d + + if not exists(output): + makedirs(output) + + json_out = f"{prefix}_extracted.json" if prefix else "extracted.json" + root_out = f"{prefix}_extracted.root" if prefix else "extracted.root" + json_out = abspath(join(output, json_out)) + root_out = abspath(join(output, root_out)) + + if not extract(files, root_out, include_file_directories=include_directories, add_if_exists=add_if_exists, reference_extracted=reference_extracted, json_extracted=json_out): + return None, None + + d = None + with open(json_out, "r") as f: + d = json.load(f) + + d["label"] = label + + with open(json_out, "w") as f: + json.dump(d, f, indent=2) + + return json_out, d + + +def only_extract(args): + if not only_extract_impl(args.input, args.output, None, args.label, args.reference): + return 1 + return 0 + -def rel_val_root(files1, files2, include_root_dirs, add_to_previous, metrics_enabled, metrics_disabled, label1, label2, output_dir, no_extract=False): +def rel_val_root(d1, d2, metrics_enabled, metrics_disabled, output_dir): """ RelVal for 2 ROOT files, simply a wrapper around ReleaseValidation.C macro @@ -189,59 +265,20 @@ def rel_val_root(files1, files2, include_root_dirs, add_to_previous, metrics_ena in case of success, return the path to the JSON with computed metrics None otherwise """ - def get_files_from_list(list_filename): - """ - Quick helper - - Extract filenames from what is listed in a given file - """ - collect_files = [] - with open(list_filename, "r") as f: - for line in f: - line = line.strip() - if not line: - continue - collect_files.append(line) - return collect_files print("==> Process and compare 2 sets of files <==") - # flat ROOT files to extract to and read from during RelVal; make absolute paths so we don't confuse ourselves when running e.g. ROOT macros in different directories - file_1 = abspath(join(output_dir, "extracted_objects_1.root")) - file_2 = abspath(join(output_dir, "extracted_objects_2.root")) - - if len(files1) == 1 and files1[0][0] == "@": - files1 = get_files_from_list(files1[0][1:]) - if not files1: - print(f"ERROR: Apparently {files1[0][1:]} contains no files to be extracted.") - return None - if len(files2) == 1 and files2[0][0] == "@": - files2 = get_files_from_list(files2[0][1:]) - if not files2: - print(f"ERROR: Apparently {files2[0][1:]} contains no files to be extracted.") - return None - - # prepare the output directory - if not exists(output_dir): - makedirs(output_dir) - log_file_rel_val = join(abspath(output_dir), "rel_val.log") - - if no_extract: - # in this case we expect the input files to be what we would otherwise extract first - if len(files1) != 1 or len(files2) != 1: - print(f"ERROR: --no-extract option was passed and expecting list of files to be of length 1 each. However, received lengths of {len(files1)} and {len(files2)}") - return 1 - file_1 = abspath(files1[0]) - file_2 = abspath(files2[0]) - elif not extract(files1, file_1, include_root_dirs, add_to_previous) or not extract(files2, file_2, include_root_dirs, add_to_previous, reference_extracted=file_1): - return None + file_1 = d1["path"] + file_2 = d2["path"] # RelVal on flattened files metrics_enabled = ";".join(metrics_enabled) if metrics_enabled else "" metrics_disabled = ";".join(metrics_disabled) if metrics_disabled else "" - cmd = f"\\(\\\"{file_1}\\\",\\\"{file_2}\\\",\\\"{metrics_enabled}\\\",\\\"{metrics_disabled}\\\",\\\"{label1}\\\",\\\"{label2}\\\"\\)" + cmd = f"\\(\\\"{file_1}\\\",\\\"{file_2}\\\",\\\"{metrics_enabled}\\\",\\\"{metrics_disabled}\\\"\\)" cmd = f"root -l -b -q {ROOT_MACRO_RELVAL}{cmd}" + output_dir = abspath(output_dir) + log_file_rel_val = join(output_dir, "rel_val.log") print("Running RelVal on extracted objects") ret = run_macro(cmd, log_file_rel_val, cwd=output_dir) @@ -352,6 +389,8 @@ def interpret_results(result, metric): need_apply = False is_inspect = False + json1 = None + json2 = None if hasattr(args, "json_path"): # this comes from the inspect command is_inspect = True @@ -364,18 +403,25 @@ def interpret_results(result, metric): include_patterns, exclude_patterns = (None, None) if args.add: print(f"NOTE: Extracted objects will be added to existing ones in case there was already a RelVal at {args.output}.\n") - json_path = rel_val_root(args.input1, args.input2, args.include_dirs, args.add, args.enable_metric, args.disable_metric, args.labels[0], args.labels[1], args.output, args.no_extract) + + json1 = only_extract_impl(args.input1, args.output, args.labels[0], args.include_dirs, args.add, prefix="1", reference_extracted=None) + json2 = only_extract_impl(args.input2, args.output, args.labels[1], args.include_dirs, args.add, prefix="2", reference_extracted=json1[1]["path"]) + if None in json1 or None in json2: + print("ERROR: Something went wrong during the extraction") + return 1 + json_path = rel_val_root(json1[1], json2[1], args.enable_metric, args.disable_metric, args.output) if json_path is None: + print("ERROR: Problem during RelVal") return 1 - annotations = {"batch_i": [abspath(p) for p in args.input1], - "batch_j": [abspath(p) for p in args.input2]} + annotations = {"json_path_1": json1[0], + "json_path_2": json2[0]} rel_val = load_rel_val(json_path, include_patterns, exclude_patterns, args.enable_metric, args.disable_metric) if need_apply or args.use_values_as_thresholds or args.default_threshold or args.regions: evaluator = initialise_evaluator(rel_val, args.use_values_as_thresholds, args.default_threshold, args.margin_threshold, args.combine_thresholds, args.regions) rel_val.apply(evaluator) - # assign interpretations to the results we got + # assign interpretations to the results we got rel_val.interpret(interpret_results) def filter_on_interpretations(result): @@ -389,15 +435,29 @@ def filter_on_interpretations(result): # if this comes from inspecting, there will be the annotations from the rel-val before that ==> re-write it rel_val.write(join(args.output, "Summary.json"), annotations=annotations or rel_val.annotations[0]) - if is_inspect: - copy_overlays(rel_val, join(dirname(json_path), "overlayPlots"), join(args.output, "overlayPlots")) + print_summary(rel_val, variables.REL_VAL_SEVERITIES, long=args.print_long) if not args.no_plot: + print("Now plotting...") # plot various different figures for user inspection plot_pie_charts(rel_val, variables.REL_VAL_SEVERITIES, variables.REL_VAL_SEVERITY_COLOR_MAP, args.output) plot_compare_summaries((rel_val,), args.output) plot_summary_grid(rel_val, variables.REL_VAL_SEVERITIES, variables.REL_VAL_SEVERITY_COLOR_MAP, args.output) - print_summary(rel_val, variables.REL_VAL_SEVERITIES, long=args.print_long) + + if is_inspect: + if annotations_inspect := rel_val.annotations: + annotations_inspect = annotations_inspect[0] + d1 = get_extract_json_info(annotations_inspect["json_path_1"]) + d2 = get_extract_json_info(annotations_inspect["json_path_2"]) + else: + d1 = json1[1] + d2 = json2[1] + + if d1 and d2: + overlay_plots_out = join(args.output, "overlayPlots") + if not exists(overlay_plots_out): + makedirs(overlay_plots_out) + plot_overlays(rel_val, d1, d2, overlay_plots_out) return 0 @@ -598,6 +658,15 @@ def print_header(): PRINT_PARSER.add_argument("--object-names", dest="object_names", action="store_true") PRINT_PARSER.set_defaults(func=print_simple) +EXTRACT_PARSER = SUB_PARSERS.add_parser("extract", parents=[COMMON_VERBOSITY_PARSER]) +EXTRACT_PARSER.add_argument("--input", nargs="*", help="Set of input files to be extracted", required=True) +EXTRACT_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val_extracted") +EXTRACT_PARSER.add_argument("--prefix", "-p", help="prefix to prepend to output files") +EXTRACT_PARSER.add_argument("--label", "-l", help="label to be assigned", required=True) +EXTRACT_PARSER.add_argument("--reference", "-r", help="path to a reference extraction file (useful to have same binning when TTrees are extracted)") +EXTRACT_PARSER.set_defaults(func=only_extract) + + def main(): """entry point when run directly from command line""" args = PARSER.parse_args() diff --git a/RelVal/run/run_data_rel_val.sh b/RelVal/run/run_data_rel_val.sh new file mode 100755 index 000000000..bb3895483 --- /dev/null +++ b/RelVal/run/run_data_rel_val.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +[[ "${#}" != "2" ]] && { echo "ERROR: Need 2 files as arguments" ; exit 1 ; } + +if [[ -z ${O2DPG_ROOT+x} ]] ; then + echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment." +else + FILE1=${1} + FILE2=${2} + DATE=$(date '+%Y-%m-%d_%H-%M') + OUTPUT=rel_val_${DATE} + OUTPUT_ALL=${OUTPUT}/rel_val_all + OUTPUT_ALL_BAD=${OUTPUT_ALL}_BAD + OUTPUT_DET="${OUTPUT}/rel_val_det" + rm -r ${OUTPUT} 2>/dev/null + + echo "Full RelVal to ${OUTPUT_ALL}" + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${FILE1} -j ${FILE2} -o ${OUTPUT_ALL} # --labels label1 label2 + echo "Extract BAD from ${OUTPUT_ALL} and write to ${OUTPUT_ALL_BAD}" + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${OUTPUT_ALL} --output ${OUTPUT_ALL_BAD} --interpretations BAD + echo "RelVal per detector..." + for det in CPV EMC FDD FT0 FV0 GLO ITS MCH MFT MID PHS TOF TPC TRD ZDC ; do + echo "...for ${det} to ${OUTPUT_DET}_${det}, checking for include pattern int_${det}_ ; the latter might need to be changed depending on the internal file structure of the QC ROOT file" + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${OUTPUT_ALL} --output ${OUTPUT_DET}_${det} --include-patterns "int_${det}_" + done +fi diff --git a/RelVal/ExtractAndFlatten.C b/RelVal/utils/ExtractAndFlatten.C similarity index 71% rename from RelVal/ExtractAndFlatten.C rename to RelVal/utils/ExtractAndFlatten.C index 09de2c645..5a1ae7dde 100644 --- a/RelVal/ExtractAndFlatten.C +++ b/RelVal/utils/ExtractAndFlatten.C @@ -1,10 +1,10 @@ -void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::string const& basedOnTree = "", std::string const& currentPrefix = "", std::vector* includeDirs = nullptr); -void ExtractTree(TTree* tree, TDirectory* outDir, std::string const& basedOnTree = "", std::string const& currentPrefix = ""); -void ExtractFromMonitorObjectCollection(o2::quality_control::core::MonitorObjectCollection* o2MonObjColl, TDirectory* outDir, std::string const& currentPrefix = ""); -void WriteHisto(TH1* obj, TDirectory* outDir, std::string const& currentPrefix = ""); -void WriteTEfficiency(TEfficiency* obj, TDirectory* outDir, std::string const& currentPrefix = ""); -void WriteToDirectory(TH1* histo, TDirectory* dir, std::string const& prefix = ""); -bool WriteObject(TObject* o, TDirectory* outDir, std::string const& currentPrefix = ""); +void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::vector& collectNames, std::string const& basedOnTree = "", std::string const& currentPrefix = "", std::vector* includeDirs = nullptr); +void ExtractTree(TTree* tree, TDirectory* outDir, std::vector& collectNames, std::string const& basedOnTree = "", std::string const& currentPrefix = ""); +void ExtractFromMonitorObjectCollection(o2::quality_control::core::MonitorObjectCollection* o2MonObjColl, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix = ""); +void WriteHisto(TH1* obj, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix = ""); +void WriteTEfficiency(TEfficiency* obj, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix = ""); +void WriteToDirectory(TH1* histo, TDirectory* dir, std::vector& collectNames, std::string const& prefix = ""); +bool WriteObject(TObject* o, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix = ""); // use this potentially to write histograms from TTree::Draw to TDirectory* BUFFER_DIR = nullptr; @@ -19,7 +19,7 @@ bool checkFileOpen(TFile* file) // outputFilename: Where to store histograms of flattened output // basedOnTree: This is in principle only needed for TTrees to determine the x-axis range and binning -int ExtractAndFlatten(std::string const& filename, std::string const& outputFilename, std::string const& basedOnTree = "", std::string const& includeDirsString = "") +int ExtractAndFlatten(std::string const& filename, std::string const& outputFilename, std::string const& basedOnTree = "", std::string const& includeDirsString = "", std::string const& outJson = "") { gROOT->SetBatch(); @@ -52,17 +52,36 @@ int ExtractAndFlatten(std::string const& filename, std::string const& outputFile return 1; } TFile extractedFile(outputFilename.c_str(), "UPDATE"); - ExtractAndFlattenDirectory(&inFile, &extractedFile, basedOnTree, "", includeDirs); + // collect the names so that we can dump them to a JSON file afterwards + std::vector collectNames; + ExtractAndFlattenDirectory(&inFile, &extractedFile, collectNames, basedOnTree, "", includeDirs); inFile.Close(); extractedFile.Close(); + if (!outJson.empty()) { + std::ofstream jsonout(outJson.c_str()); + jsonout << "{\n" << " \"path\": " << std::filesystem::absolute(outputFilename) << ","; + jsonout << "\n" << " \"objects\": [\n"; + int mapIndex = 0; + int mapSize = collectNames.size(); + for (auto& name : collectNames) { + jsonout << "\"" << name << "\""; + if (++mapIndex < mapSize) { + // this puts a comma except for the very last entry + jsonout << ",\n"; + } + } + jsonout << "\n ]\n}"; + jsonout.close(); + } return 0; } // writing a TObject to a TDirectory -void WriteToDirectory(TH1* histo, TDirectory* dir, std::string const& prefix) +void WriteToDirectory(TH1* histo, TDirectory* dir, std::vector& collectNames, std::string const& prefix) { std::string name = prefix + histo->GetName(); + collectNames.push_back(name); histo->SetName(name.c_str()); auto hasObject = (TH1*)dir->Get(name.c_str()); @@ -117,7 +136,7 @@ bool checkIncludePath(std::string thisPath, std::vector*& includeDi } // Read from a given input directory and write everything found there (including sub directories) to a flat output directory -void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::string const& basedOnTree, std::string const& currentPrefix, std::vector* includeDirs) +void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::vector& collectNames, std::string const& basedOnTree, std::string const& currentPrefix, std::vector* includeDirs) { if (!checkIncludePath(inDir->GetPath(), includeDirs)) { @@ -129,25 +148,25 @@ void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::stri auto obj = key->ReadObj(); if (auto nextInDir = dynamic_cast(obj)) { // recursively scan TDirectory - ExtractAndFlattenDirectory(nextInDir, outDir, basedOnTree, currentPrefix + nextInDir->GetName() + "_", includeDirs); + ExtractAndFlattenDirectory(nextInDir, outDir, collectNames, basedOnTree, currentPrefix + nextInDir->GetName() + "_", includeDirs); } else if (auto qcMonitorCollection = dynamic_cast(obj)) { auto qcMonPath = std::string(inDir->GetPath()) + "/" + qcMonitorCollection->GetName(); auto includeDirsCache = includeDirs; if (!checkIncludePath(qcMonPath, includeDirsCache)) { continue; } - ExtractFromMonitorObjectCollection(qcMonitorCollection, outDir, currentPrefix); + ExtractFromMonitorObjectCollection(qcMonitorCollection, outDir, collectNames, currentPrefix); } else if (auto tree = dynamic_cast(obj)) { - ExtractTree(tree, outDir, basedOnTree, currentPrefix); + ExtractTree(tree, outDir, collectNames, basedOnTree, currentPrefix); } else { - if (!WriteObject(obj, outDir, currentPrefix)) { + if (!WriteObject(obj, outDir, collectNames, currentPrefix)) { std::cerr << "Cannot handle object " << obj->GetName() << " which is of class " << key->GetClassName() << "\n"; } } } } -void ExtractTree(TTree* tree, TDirectory* outDir, std::string const& basedOnTree, std::string const& currentPrefix) +void ExtractTree(TTree* tree, TDirectory* outDir, std::vector& collectNames, std::string const& basedOnTree, std::string const& currentPrefix) { const std::vector acceptedLeafTypes{"char", "int", "float", "double"}; TIter next(tree->GetListOfLeaves()); @@ -211,18 +230,18 @@ void ExtractTree(TTree* tree, TDirectory* outDir, std::string const& basedOnTree std::cerr << "WARNING: Cannot draw TLeaf " << ln << "\n"; continue; } - WriteObject(currentHist, outDir); + WriteObject(currentHist, outDir, collectNames); } BUFFER_DIR->Clear(); } // extract everything from a o2::quality_control::core::MonitorObjectCollection object -void ExtractFromMonitorObjectCollection(o2::quality_control::core::MonitorObjectCollection* o2MonObjColl, TDirectory* outDir, std::string const& currentPrefix) +void ExtractFromMonitorObjectCollection(o2::quality_control::core::MonitorObjectCollection* o2MonObjColl, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix) { std::cout << "--- Process o2 Monitor Object Collection " << o2MonObjColl->GetName() << " ---\n"; int nProcessed{}; for (int j = 0; j < o2MonObjColl->GetEntries(); j++) { - if (WriteObject(o2MonObjColl->At(j), outDir, currentPrefix + o2MonObjColl->GetName() + "_")) { + if (WriteObject(o2MonObjColl->At(j), outDir, collectNames, currentPrefix + o2MonObjColl->GetName() + "_")) { nProcessed++; } } @@ -240,18 +259,18 @@ void adjustName(TObject* o) } // decide which concrete function to call to write the given object -bool WriteObject(TObject* o, TDirectory* outDir, std::string const& currentPrefix) +bool WriteObject(TObject* o, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix) { if (auto monObj = dynamic_cast(o)) { - return WriteObject(monObj->getObject(), outDir, currentPrefix); + return WriteObject(monObj->getObject(), outDir, collectNames, currentPrefix); } adjustName(o); if (auto eff = dynamic_cast(o)) { - WriteTEfficiency(eff, outDir, currentPrefix); + WriteTEfficiency(eff, outDir, collectNames, currentPrefix); return true; } if (auto hist = dynamic_cast(o)) { - WriteHisto(hist, outDir, currentPrefix); + WriteHisto(hist, outDir, collectNames, currentPrefix); return true; } std::cerr << "WARNING: Cannot process object " << o->GetName() << "\n"; @@ -259,13 +278,13 @@ bool WriteObject(TObject* o, TDirectory* outDir, std::string const& currentPrefi } // Implementation to write a TH1 -void WriteHisto(TH1* hA, TDirectory* outDir, std::string const& currentPrefix) +void WriteHisto(TH1* hA, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix) { - WriteToDirectory(hA, outDir, currentPrefix); + WriteToDirectory(hA, outDir, collectNames, currentPrefix); } // Implementation to extract TH1 from TEfficieny and write them -void WriteTEfficiency(TEfficiency* hEff, TDirectory* outDir, std::string const& currentPrefix) +void WriteTEfficiency(TEfficiency* hEff, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix) { // should I further develop that? // separate numerator and denominator of the efficiency. // NOTE These have no directory assigned -> GOOD @@ -281,9 +300,9 @@ void WriteTEfficiency(TEfficiency* hEff, TDirectory* outDir, std::string const& hEffWrite->SetTitle(Form("%s", hEff->GetTitle())); hEffWrite->Divide(hEffNumerator, hEffDenominator, 1., 1., "B"); - WriteToDirectory(hEffNumerator, outDir, currentPrefix); - WriteToDirectory(hEffDenominator, outDir, currentPrefix); - WriteToDirectory(hEffWrite, outDir, currentPrefix); + WriteToDirectory(hEffNumerator, outDir, collectNames, currentPrefix); + WriteToDirectory(hEffDenominator, outDir, collectNames, currentPrefix); + WriteToDirectory(hEffWrite, outDir, collectNames, currentPrefix); delete hEffNumerator; delete hEffDenominator; diff --git a/RelVal/ReleaseValidation.C b/RelVal/utils/ReleaseValidation.C similarity index 91% rename from RelVal/ReleaseValidation.C rename to RelVal/utils/ReleaseValidation.C index f66edbe47..cbebc9eae 100644 --- a/RelVal/ReleaseValidation.C +++ b/RelVal/utils/ReleaseValidation.C @@ -3,7 +3,6 @@ #include #include #include -#include "PlotOverlays.C" #include "ReleaseValidationMetrics.C" // define a global epsilon @@ -53,7 +52,7 @@ int isEmptyHisto(TH1* h) // 6) select if files have to be taken from the grid or not // 7) choose if specific critic plots have to be saved in a second .pdf file -int ReleaseValidation(std::string const& filename1, std::string const& filename2, std::string const& withMetrics="", std::string const& withoutMetrics="", std::string const& labelA="batch_i", std::string const& labelB="batch_j") +int ReleaseValidation(std::string const& filename1, std::string const& filename2, std::string const& withMetrics="", std::string const& withoutMetrics="") { gROOT->SetBatch(); @@ -125,23 +124,7 @@ int ReleaseValidation(std::string const& filename1, std::string const& filename2 auto ncCode = CheckComparable(hA, hB); auto areComparable = NCCodes::isComparable(ncCode); - TLegend legendMetricsOverlayPlot(0.6, 0.6, 0.9, 0.8); - legendMetricsOverlayPlot.SetBorderSize(1); - legendMetricsOverlayPlot.SetFillStyle(0); - - std::vector metricResults; - metricRunner.evaluate(hA, hB, ncCode, metricResults); - for (auto& metricResult : metricResults) { - if (metricResult.comparable) { - legendMetricsOverlayPlot.AddEntry((TObject*)nullptr, Form("%s = %f", metricResult.name.c_str(), metricResult.value), ""); - } - } - - if (isEmptyHisto(hA) == 2 || isEmptyHisto(hB) == 2) { - std::cerr << "WARNING: Cannot draw objects due to the fact that all entries are in under- or overflow bins\n"; - continue; - } - PlotOverlayAndRatio({hA, hB}, {labelA, labelB}, "overlayPlots", &legendMetricsOverlayPlot); + metricRunner.evaluate(hA, hB, ncCode); nComparisons++; } @@ -155,6 +138,9 @@ int ReleaseValidation(std::string const& filename1, std::string const& filename2 WriteToJsonFromMap(metricRunner); + extractedFile1.Close(); + extractedFile2.Close(); + return 0; } diff --git a/RelVal/ReleaseValidationMetrics.C b/RelVal/utils/ReleaseValidationMetrics.C similarity index 97% rename from RelVal/ReleaseValidationMetrics.C rename to RelVal/utils/ReleaseValidationMetrics.C index 04c7f0a9b..40cff7d7c 100644 --- a/RelVal/ReleaseValidationMetrics.C +++ b/RelVal/utils/ReleaseValidationMetrics.C @@ -130,16 +130,14 @@ struct MetricRunner } } - void evaluate(TH1* hA, TH1* hB, NCCodes::CODE code, std::vector& metricResultsReturn) + void evaluate(TH1* hA, TH1* hB, NCCodes::CODE code) { - metricResultsReturn.clear(); for (auto& metric : metricsEnabled) { if (!metric) { // here is a nullptr so it is not active continue; } metricResults.push_back(metric->evaluate(hA, hB, code)); - metricResultsReturn.push_back(metricResults.back()); } } diff --git a/RelVal/o2dpg_release_validation_plot.py b/RelVal/utils/o2dpg_release_validation_plot.py similarity index 82% rename from RelVal/o2dpg_release_validation_plot.py rename to RelVal/utils/o2dpg_release_validation_plot.py index 38d788382..17b774dab 100755 --- a/RelVal/o2dpg_release_validation_plot.py +++ b/RelVal/utils/o2dpg_release_validation_plot.py @@ -7,6 +7,7 @@ from os import environ import importlib.util from itertools import product +import re import numpy as np import matplotlib.pyplot as plt from matplotlib.colors import LinearSegmentedColormap @@ -14,12 +15,17 @@ O2DPG_ROOT = environ.get("O2DPG_ROOT") -spec = importlib.util.spec_from_file_location("o2dpg_release_validation_utils", join(O2DPG_ROOT, "RelVal", '.', 'o2dpg_release_validation_utils.py')) +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_utils", join(O2DPG_ROOT, "RelVal", "utils", '.', 'o2dpg_release_validation_utils.py')) o2dpg_release_validation_utils = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_utils) sys.modules["o2dpg_release_validation_utils"] = o2dpg_release_validation_utils from o2dpg_release_validation_utils import count_interpretations +spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot_root", join(O2DPG_ROOT, "RelVal", "utils", '.', 'o2dpg_release_validation_plot_root.py')) +o2dpg_release_validation_plot_root = importlib.util.module_from_spec(spec) +spec.loader.exec_module(o2dpg_release_validation_plot_root) +sys.modules["o2dpg_release_validation_plot_root"] = o2dpg_release_validation_plot_root +from o2dpg_release_validation_plot_root import plot_overlays_root, plot_overlays_root_no_rel_val def plot_pie_charts(rel_val, interpretations, interpretation_colors, out_dir, title="", get_figure=False): @@ -31,6 +37,9 @@ def plot_pie_charts(rel_val, interpretations, interpretation_colors, out_dir, ti labels = [] object_names, results = rel_val.get_result_per_metric_and_test(metric_name, test_name) + if not len(object_names): + continue + for interpretation in interpretations: n_objects = len(object_names[count_interpretations(results, interpretation)]) if not n_objects: @@ -105,6 +114,8 @@ def plot_compare_summaries(rel_vals, out_dir, *, labels=None, get_figure=False): if labels is given, it needs to have the same length as summaries """ + print("==> Plot metric values <==") + figures = [] if not labels: @@ -118,14 +129,18 @@ def plot_compare_summaries(rel_vals, out_dir, *, labels=None, get_figure=False): for metric_name, test_name in product(metric_names, test_names): figure, ax = plt.subplots(figsize=(20, 20)) + plot_this = False for rel_val, label in zip(rel_vals, labels): object_names, results = rel_val.get_result_per_metric_and_test(metric_name, test_name) values = [result.value for result in results] means = [result.mean for result in results] if not values: continue + plot_this = True ax.plot(object_names, values, label=f"values_{label}") ax.plot(object_names, means, label=f"test_means_{label}") + if not plot_this: + continue ax.legend(loc="best", fontsize=20) ax.tick_params("both", labelsize=20) ax.tick_params("x", rotation=90) @@ -138,3 +153,13 @@ def plot_compare_summaries(rel_vals, out_dir, *, labels=None, get_figure=False): plt.close(figure) if get_figure: return figures + + +def plot_overlays(rel_val, file_config_map1, file_config_map2, out_dir, plot_regex=None): + print("==> Plot overlays <==") + plot_overlays_root(rel_val, file_config_map1, file_config_map2, out_dir, plot_regex) + + +def plot_overlays_no_rel_val(file_configs, out_dir): + print("==> Plot overlays <==") + plot_overlays_root_no_rel_val(file_configs, out_dir) diff --git a/RelVal/utils/o2dpg_release_validation_plot_root.py b/RelVal/utils/o2dpg_release_validation_plot_root.py new file mode 100755 index 000000000..98bd20ae3 --- /dev/null +++ b/RelVal/utils/o2dpg_release_validation_plot_root.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# +# Definition common functionality + +from os.path import join, exists +from os import remove +from math import log10, pow +import re + +from ctypes import c_char_p +from ROOT import gSystem, TFile, TCanvas, TPad, TLegend, TH2, TH3, TText, TPaveText, kWhite, kRed, kBlue, kGreen, kMagenta, kCyan, kOrange, kYellow, TProfile + + +def style_histograms(histograms): + if isinstance(histograms[0], (TH2, TH3)): + return + colors = (kRed + 2, kBlue - 4, kGreen + 3, kMagenta + 1, kCyan + 2, kOrange + 5, kYellow - 6) + linestyles = (1, 10, 2, 9, 8, 7) + + for i, h in enumerate(histograms): + h.SetLineStyle(linestyles[i % len(linestyles)]) + h.SetLineColor(colors[i % len(colors)]) + h.SetLineWidth(1) + +def findRangeNotEmpty1D(histogram): + axis = histogram.GetXaxis() + minX = axis.GetBinLowEdge(1) + maxX = axis.GetBinUpEdge(axis.GetNbins()) + + for i in range(1, axis.GetNbins() + 1): + # go from left to right to find the first non-empty bin + if histogram.GetBinContent(i) != 0: + minX = axis.GetBinLowEdge(i) + break + + for i in range(axis.GetNbins(), 0, -1): + # go from right to left to find the last non-empty bin + if histogram.GetBinContent(i) != 0: + maxX = axis.GetBinUpEdge(i) + break + + return minX, maxX + + +def adjust_axis_text(axis, label_size, title_size): + axis.SetLabelFont(43) + axis.SetTitleFont(43) + axis.SetLabelSize(label_size) + axis.SetTitleSize(title_size) + + +def make_frame(pad, histograms): + integralRef = histograms[0].Integral() + shouldBeLog = False + minY = histograms[0].GetMinimum(0) + maxY = histograms[0].GetMaximum() + + minX, maxX = findRangeNotEmpty1D(histograms[0]) + + # find minima and maxima + for h in histograms[1:]: + minY = min(h.GetMinimum(0), minY) + maxY = max(h.GetMaximum(), maxY) + + minXNext, maxXNext = findRangeNotEmpty1D(h) + minX = min(minX, minXNext) + maxX = max(maxX, maxXNext) + + integral = h.Integral() + if (integralRef > 0 and integral / integralRef > 100) or (integral > 0 and integralRef / integral > 100): + # decide whether to do a log plot + shouldBeLog = True + + # finalise the y-axis limits + if shouldBeLog: + margin = log10(maxY / minY) + minY = minY / pow(10, margin * 0.1) + maxY = maxY * pow(10, margin * 0.3) + else: + margin = 0.2 * (maxY - minY) + maxY += 3 * margin + minY -= max(0., margin) + + if histograms[0].GetXaxis().IsAlphanumeric(): + alphanumericFrame = histograms[0].Clone() + alphanumericFrame.Reset("ICEMS") + return alphanumericFrame, shouldBeLog + + frame = pad.DrawFrame(minX, minY, maxX, maxY, histograms[0].GetYaxis().GetTitle()) + + return frame, shouldBeLog + + +def plot_single_overlay_1d(histograms, more_objects, out_path, *args): + + ratios = [] + denominator = histograms[0] + if isinstance(denominator, TProfile): + return + for h in histograms[1:]: + ratio = h.Clone() + ratio.SetDirectory(0) + ratio.Divide(h, denominator, 1, 1, "B") + ratios.append(ratio) + + c = TCanvas("overlay", "", 800, 800) + c.cd() + + nominalPad = TPad("nominalPad", "nominalPad", 0, 0.3, 1., 1.) + nominalPad.SetBottomMargin(0) + ratioPad = TPad("ratioPad", "ratioPad", 0, 0.05, 1. ,0.32) + ratioPad.SetTopMargin(0) + ratioPad.SetBottomMargin(0.2) + + nominalPad.Draw() + ratioPad.Draw() + + nominalPad.cd() + nominalFrame, logY = make_frame(nominalPad, histograms) + yAxis = nominalFrame.GetYaxis() + yAxis.ChangeLabel(1, -1, -1, -1, -1, -1, " ") + + adjust_axis_text(yAxis, 20, 20) + adjust_axis_text(nominalFrame.GetXaxis(), 0, 0) + nominalFrame.Draw("*") + for h in histograms: + h.Draw("same E hist") + for mo in more_objects: + mo.Draw("same") + + if logY: + nominalPad.SetLogy() + + ratioPad.cd() + ratioFrame, logY = make_frame(ratioPad, ratios) + axis = ratioFrame.GetXaxis() + axis.SetTitle(histograms[0].GetXaxis().GetTitle()) + adjust_axis_text(axis, 20, 20) + axis = ratioFrame.GetYaxis() + axis.SetTitle("ratio") + adjust_axis_text(axis, 20, 20) + + ratioFrame.Draw("*") + for ratio in ratios: + ratio.Draw("same") + if logY: + ratioFrame.SetLogy() + + c.SaveAs(out_path) + c.Close() + + +def plot_single_overlay_2d(histograms, more_objects, out_path, labels=None): + + n_histograms = len(histograms) + if not labels: + labels = [f"label_{i}" for i in range(n_histograms)] + + c = TCanvas("overlay", "", 2400, 800 * (n_histograms-1)) + c.Divide(3, n_histograms - 1) + c.cd(1) + histograms[0].SetTitle(histograms[0].GetTitle() + f"({labels[0]})") + histograms[0].SetStats(0); + histograms[0].Draw("colz"); + + keep_elements = [] + if histograms[0].GetEntries() == 0: + t1 = TText(0.5, 0.5, "EMPTY") + keep_elements.append(t1) + t1.SetNDC() + t1.Draw() + + ratios = [] + + for i, h in enumerate(histograms[1:], start=1): + ratio = h.Clone() + ratio.SetDirectory(0) + ratios.append(ratio) + ratio.SetTitle(f"{h.GetTitle()} ({labels[i]} / {labels[0]})") + ratio.SetStats(0) + ratio.Divide(histograms[0]) + h.SetStats(0) + h.SetTitle(f"{h.GetTitle()} ({labels[i]})") + + c.cd(i * 3 - 1) + h.Draw("colz") + if h.GetEntries() == 0: + t1 = TText(0.5, 0.5, "EMPTY") + t1.SetNDC() + t1.Draw() + keep_elements.append(t1) + + c.cd(i * 3) + ratio.Draw("colz") + + c.cd(3) + for mo in more_objects: + mo.Draw("same") + + c.SaveAs(out_path) + c.Close() + + +def plot_overlays_root(rel_val, file_config_map1, file_config_map2, out_dir, plot_regex=None): + + + file1 = TFile(file_config_map1["path"], "READ") + file2 = TFile(file_config_map2["path"], "READ") + + label1 = file_config_map1["label"] + label2 = file_config_map2["label"] + + plot_log_file = join(out_dir, "overlay_plotting.log") + if exists(plot_log_file): + remove(plot_log_file) + + for object_name, metrics, results in rel_val.yield_metrics_results_per_object(): + if plot_regex is not None and not re.search(object_name, plot_regex): + continue + + metric_legend_entries = {} + for metric, result in zip(metrics, results): + if metric.name not in metric_legend_entries: + value = "NONE" if not metric.comparable else f"{metric.value}" + metric_legend_entries[metric.name] = value + if result is None: + continue + metric_legend_entries[metric.name] += f", {result.interpretation}" + + h1 = file1.Get(object_name) + h2 = file2.Get(object_name) + + more_objects = [] + plot_func = plot_single_overlay_2d + metrics_box = TPaveText(0.15, 0.7, 0.4, 0.9, "brNDC") + metrics_box.SetTextFont(43) + metrics_box.SetTextSize(20) + metrics_box.SetBorderSize(0) + more_objects.append(metrics_box) + if not isinstance(h1, (TH2, TH3)): + plot_func = plot_single_overlay_1d + metrics_box.SetFillStyle(0) + style_histograms([h1, h2]) + legend_labels = TLegend(0.65, 0.7, 0.9, 0.9) + legend_labels.SetFillStyle(0) + legend_labels.SetBorderSize(0) + legend_labels.SetTextFont(43) + legend_labels.SetTextSize(20) + legend_labels.AddEntry(h1, label1) + legend_labels.AddEntry(h2, label2) + more_objects.append(legend_labels) + else: + metrics_box.SetFillColor(kWhite) + + legend_metrics = TLegend(0.15, 0.7, 0.4, 0.9) + legend_metrics.SetBorderSize(0) + legend_metrics.SetFillStyle(0) + for key, value in metric_legend_entries.items(): + metrics_box.AddText(f"{key} = {value}") + + out_path = join(out_dir, f"{object_name}.png") + gSystem.RedirectOutput(join(out_dir, "overlay_plotting.log"), "a") + plot_func([h1, h2], more_objects, out_path, [label1, label2]) + gSystem.RedirectOutput(c_char_p(0)) + + print(f"INFO: Log file for overlay plotting at {plot_log_file}") + + +def plot_overlays_root_no_rel_val(file_configs, out_dir): + gSystem.RedirectOutput(join(out_dir, "overlay_plotting.log"), "w") + + all_names = [] + labels = [] + files = [] + names_per_file = [] + for fc in file_configs: + all_names.extend(fc["objects"]) + labels.append(fc["label"]) + names_per_file.append(fc["objects"]) + files.append(TFile(fc["path"], "READ")) + + all_names = list(set(all_names)) + for name in all_names: + histograms = [] + current_labels = [] + + for object_names, label, f in zip(labels, files): + if name not in object_names: + continue + histograms.append(f.Get(name)) + current_labels.append(label) + + if not histograms: + continue + + more_objects = [] + plot_func = plot_single_overlay_2d + if not isinstance(histograms[0], (TH2, TH3)): + plot_func = plot_single_overlay_1d + style_histograms(histograms) + legend_labels = TLegend(0.65, 0.7, 0.9, 0.9) + legend_labels.SetFillStyle(0) + legend_labels.SetBorderSize(0) + legend_labels.SetTextFont(43) + legend_labels.SetTextSize(20) + for h, cl in zip(histograms, current_labels): + legend_labels.AddEntry(h, cl) + more_objects.append(legend_labels) + + out_path = join(out_dir, f"{name}.png") + plot_func(histograms, more_objects, out_path, labels) + + gSystem.RedirectOutput(c_char_p(0)) diff --git a/RelVal/o2dpg_release_validation_utils.py b/RelVal/utils/o2dpg_release_validation_utils.py similarity index 97% rename from RelVal/o2dpg_release_validation_utils.py rename to RelVal/utils/o2dpg_release_validation_utils.py index 235b6116f..873d4de44 100755 --- a/RelVal/o2dpg_release_validation_utils.py +++ b/RelVal/utils/o2dpg_release_validation_utils.py @@ -496,6 +496,24 @@ def get_result_matrix_objects_metrics(self, test_index): return metric_names, object_names, np.reshape(results[idx], (len(object_names), len(metric_names))) + def yield_metrics_results_per_object(self): + results = None + if self.results is not None: + mask = self.result_filter_mask + idx = self.results_to_metrics_idx[mask] + object_names = np.take(self.object_names, idx) + metrics = np.take(self.metrics, idx) + results = self.results[mask] + else: + object_names = self.object_names + metrics = self.metrics + + for object_name in np.unique(object_names): + mask = object_names == object_name + yield_metrics = metrics[mask] + yield_results = results[mask] if results is not None else np.array([None] * len(yield_metrics)) + yield object_name, yield_metrics, yield_results + def write(self, filepath, annotations=None): all_objects = [] @@ -525,7 +543,7 @@ def make_dict_exclude_results(object_name, metric, *args): RelVal.KEY_ANNOTATIONS: annotations} with open(filepath, "w") as f: - json.dump(final_dict, f) + json.dump(final_dict, f, indent=2) def get_summaries_or_from_file(in_objects): diff --git a/RelVal/o2dpg_release_validation_variables.py b/RelVal/utils/o2dpg_release_validation_variables.py similarity index 100% rename from RelVal/o2dpg_release_validation_variables.py rename to RelVal/utils/o2dpg_release_validation_variables.py From eeb18eaafae95785ef8790e22644a0fa40945963 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 18:38:13 +0100 Subject: [PATCH 0265/1239] More tuning for async workflow on EPNs --- DATA/production/workflow-multiplicities.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index fb0580566..81db5f102 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -106,7 +106,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then N_ITSTRK=12 export DPL_SMOOTH_RATE_LIMITING=1 elif [[ $OPTIMIZED_PARALLEL_ASYNC == "PbPb_4gpu" ]]; then - [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=30 + [[ -z ${TIMEFRAME_RATE_LIMIT:-} ]] && TIMEFRAME_RATE_LIMIT=35 [[ -z ${SHMSIZE:-} ]] && SHMSIZE=100000000000 # SHM_LIMIT 3/4 [[ -z ${TIMEFRAME_SHM_LIMIT:-} ]] && TIMEFRAME_SHM_LIMIT=$(($SHMSIZE / 3)) NGPURECOTHREADS=8 @@ -119,16 +119,17 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then NGPUS=4 N_TPCTRK=4 # time in s: pvtx 16, tof 30, trd 82 itstpc 53 its 200 mfttr 30 tpcent 23 hmp-clus 40 (25.11.22) - N_TPCENTDEC=$(math_max $((5 * $NGPUS / 4)) 1) + N_TPCENTDEC=$(math_max $((4 * $NGPUS / 4)) 1) N_ITSTRK=$(math_max $((4 * $NGPUS / 4)) 1) - N_TPCITS=$(math_max $((5 * $NGPUS / 4)) 1) + N_TPCITS=$(math_max $((4 * $NGPUS / 4)) 1) N_MFTTRK=$(math_max $((3 * $NGPUS / 4)) 1) - N_TRDTRK=$(math_max $((9 * $NGPUS / 4)) 1) - N_TOFMATCH=$(math_max $((6 * $NGPUS / 4)) 1) + N_TRDTRK=$(math_max $((7 * $NGPUS / 4)) 1) + N_TOFMATCH=$(math_max $((5 * $NGPUS / 4)) 1) N_HMPCLUS=$(math_max $((3 * $NGPUS / 4)) 1) + N_ITSCL=3 N_AODPROD=2 N_MCHCL=9 - N_HMPMATCH=14 + N_HMPMATCH=1 N_MCHTRK=7 N_PRIMVTXMATCH=2 N_PRIMVTX=3 @@ -150,7 +151,7 @@ if [[ ! -z ${OPTIMIZED_PARALLEL_ASYNC:-} ]]; then if [[ ${OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT:-} == 1 && ${EPN_NODE_MI100:-} == 1 ]]; then TIMEFRAME_RATE_LIMIT=$(($TIMEFRAME_RATE_LIMIT * 2)) SHMSIZE=$(($SHMSIZE * 2)) - EPN_GLOBAL_SCALING=2 + EPN_GLOBAL_SCALING="3 / 2" fi elif [[ $EPNPIPELINES != 0 ]]; then NTRDTRKTHREADS=2 From 447eadefd78b8c9ac2d35cae7416e9c51f357595 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 7 Dec 2023 15:03:38 +0100 Subject: [PATCH 0266/1239] Use least significant bits of alien proc id for o2jobid / shmid --- DATA/production/common/setVarsFromALIEN_PROC_ID.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/common/setVarsFromALIEN_PROC_ID.sh b/DATA/production/common/setVarsFromALIEN_PROC_ID.sh index 3dab53050..bb2116bba 100644 --- a/DATA/production/common/setVarsFromALIEN_PROC_ID.sh +++ b/DATA/production/common/setVarsFromALIEN_PROC_ID.sh @@ -15,6 +15,6 @@ ALIEN_PROC_ID_OFFSET_INT16=${ALIEN_PROC_ID:$((ALIEN_PROC_ID_OFFSET_INT16))} # let's make them int32 or int16, but not with the max possible value (which would be 0x7fffffff and 0x7fff respectively) # but a bit less, to allow to add [0, 15] on top afterwards if needed (e.g. we usually add # the NUMAID), see https://github.com/AliceO2Group/O2DPG/pull/993#pullrequestreview-1393401475 -export O2JOBID=$((ALIEN_PROC_ID_OFFSET_INT32 & 0x7ffffff0)) -export O2JOBSHMID=$((ALIEN_PROC_ID_OFFSET_INT16 & 0x7ff0)) +export O2JOBID=$(((ALIEN_PROC_ID_OFFSET_INT32 & 0x7ffffff) * 16)) +export O2JOBSHMID=$(((ALIEN_PROC_ID_OFFSET_INT16 & 0x7ff) * 16)) echo "ALIEN_PROC_ID = $ALIEN_PROC_ID, we will set O2JOBID = $O2JOBID, SHMEMID = $SHMEMID" From 7cf2b491c46c8986a823d75540c60e9f6472bb91 Mon Sep 17 00:00:00 2001 From: shahoian Date: Fri, 8 Dec 2023 15:54:56 +0100 Subject: [PATCH 0267/1239] Build HMP_SOURCES for HMP matching depending on other settings --- DATA/common/setenv.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 1f4af758c..6c31539c9 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -174,14 +174,22 @@ fi # Assemble matching sources TRD_SOURCES= TOF_SOURCES= +HMP_SOURCES= TRACK_SOURCES= has_detectors_reco ITS TPC && has_detector_matching ITSTPC && add_comma_separated TRACK_SOURCES "ITS-TPC" has_detectors_reco TPC TRD && has_detector_matching TPCTRD && { add_comma_separated TRD_SOURCES TPC; add_comma_separated TRACK_SOURCES "TPC-TRD"; } -has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRD && { add_comma_separated TRD_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD"; } +has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && { add_comma_separated TRD_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD"; } has_detectors_reco TPC TOF && has_detector_matching TPCTOF && { add_comma_separated TOF_SOURCES TPC; add_comma_separated TRACK_SOURCES "TPC-TOF"; } has_detectors_reco ITS TPC TOF && has_detector_matching ITSTPC && has_detector_matching ITSTPCTOF && { add_comma_separated TOF_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TOF"; } has_detectors_reco TPC TRD TOF && has_detector_matching TPCTRD && has_detector_matching TPCTRDTOF && { add_comma_separated TOF_SOURCES TPC-TRD; add_comma_separated TRACK_SOURCES "TPC-TRD-TOF"; } -has_detectors_reco ITS TPC TRD TOF && has_detector_matching ITSTPCTRD && has_detector_matching ITSTPCTRDTOF && { add_comma_separated TOF_SOURCES ITS-TPC-TRD; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD-TOF"; } +has_detectors_reco ITS TPC TRD TOF && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && has_detector_matching ITSTPCTRDTOF && { add_comma_separated TOF_SOURCES ITS-TPC-TRD; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD-TOF"; } +has_detectors_reco HMP ITS TPC && has_detector_matching ITSTPC && add_comma_separated HMP_SOURCES "ITS-TPC" +has_detectors_reco HMP ITS TPC TRD && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && add_comma_separated HMP_SOURCES "ITS-TPC-TRD" +has_detectors_reco HMP ITS TPC TOF && has_detector_matching ITSTPC && has_detector_matching ITSTPCTOF && add_comma_separated HMP_SOURCES "ITS-TPC-TOF" +has_detectors_reco HMP ITS TPC TRD TOF && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && has_detector_matching ITSTPCTRDTOF && add_comma_separated HMP_SOURCES "ITS-TPC-TRD-TOF" +has_detectors_reco HMP TPC TRD && has_detector_matching TPCTRD && add_comma_separated HMP_SOURCES "TPC-TRD" +has_detectors_reco HMP TPC TOF && has_detector_matching TPCTOF && add_comma_separated HMP_SOURCES "TPC-TOF" +has_detectors_reco HMP TPC TRD TOF && has_detector_matching TPCTRD && has_detector_matching TPCTRDTOF && add_comma_separated HMP_SOURCES "TPC-TRD-TOF" has_detectors_reco MFT MCH && has_detector_matching MFTMCH && add_comma_separated TRACK_SOURCES "MFT-MCH" has_detectors_reco MCH MID && has_detector_matching MCHMID && add_comma_separated TRACK_SOURCES "MCH-MID" for det in `echo $LIST_OF_DETECTORS | sed "s/,/ /g"`; do From f5fb1400ed7a4914547f47b865232401c2e71ce5 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Fri, 8 Dec 2023 15:57:33 +0100 Subject: [PATCH 0268/1239] [RelVal] Make rel_val shell script more useful --- RelVal/run/run_data_rel_val.sh | 148 ++++++++++++++++++++++++++++----- 1 file changed, 127 insertions(+), 21 deletions(-) diff --git a/RelVal/run/run_data_rel_val.sh b/RelVal/run/run_data_rel_val.sh index bb3895483..9f33f91da 100755 --- a/RelVal/run/run_data_rel_val.sh +++ b/RelVal/run/run_data_rel_val.sh @@ -1,26 +1,132 @@ #!/bin/bash -[[ "${#}" != "2" ]] && { echo "ERROR: Need 2 files as arguments" ; exit 1 ; } - if [[ -z ${O2DPG_ROOT+x} ]] ; then - echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment." -else - FILE1=${1} - FILE2=${2} - DATE=$(date '+%Y-%m-%d_%H-%M') - OUTPUT=rel_val_${DATE} - OUTPUT_ALL=${OUTPUT}/rel_val_all - OUTPUT_ALL_BAD=${OUTPUT_ALL}_BAD - OUTPUT_DET="${OUTPUT}/rel_val_det" - rm -r ${OUTPUT} 2>/dev/null - - echo "Full RelVal to ${OUTPUT_ALL}" - ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${FILE1} -j ${FILE2} -o ${OUTPUT_ALL} # --labels label1 label2 - echo "Extract BAD from ${OUTPUT_ALL} and write to ${OUTPUT_ALL_BAD}" - ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${OUTPUT_ALL} --output ${OUTPUT_ALL_BAD} --interpretations BAD - echo "RelVal per detector..." + echo "O2DPG is not loaded, probably other packages are missing as well in this environment." + exit 1 +fi + +DATE=$(date '+%Y-%m-%d_%H-%M') +OUTPUT=rel_val_${DATE} +LOGFILE=${OUTPUT}/"rel_val.log" + +function wait_for_jobs() +{ + local n_parallel=${1:-5} + local sleep_time=${2:-2} + while true + do + n_packing=$(jobs | grep "include-patterns" | grep "Running" | wc -l ) + if (( $n_packing >= ${n_parallel} )) + then + sleep $sleep_time + else + break + fi + done +} + +rel_val_qc() +{ + local file1=${1} + local file2=${2} + local label1=${3} + local label2=${4} + local output=${OUTPUT}/QC + local output_all=${output}/all + local output_all_bad=${output_all}_BAD + local output_det="${output}/det" + rm -r ${output} 2>/dev/null + + echo "Full RelVal to ${output_all}" | tee -a ${LOGFILE} + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${file1} -j ${file2} -o ${output_all} --labels ${label1} ${label2} 2>&1 | tee -a ${LOGFILE} + echo "Extract BAD from ${output_all} and write to ${output_all_bad}" | tee -a ${LOGFILE} + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${output_all} --output ${output_all_bad} --interpretations BAD >> ${LOGFILE} 2>&1 + echo "RelVal per detector..." | tee -a ${LOGFILE} for det in CPV EMC FDD FT0 FV0 GLO ITS MCH MFT MID PHS TOF TPC TRD ZDC ; do - echo "...for ${det} to ${OUTPUT_DET}_${det}, checking for include pattern int_${det}_ ; the latter might need to be changed depending on the internal file structure of the QC ROOT file" - ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${OUTPUT_ALL} --output ${OUTPUT_DET}_${det} --include-patterns "int_${det}_" + echo "...for ${det} to ${output_det}_${det}, checking for include pattern int_${det}_ ; the latter might need to be changed depending on the internal file structure of the QC ROOT file" | tee -a ${LOGFILE} + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${output_all} --output ${output_det}_${det} --include-patterns "int_${det}_" >> ${LOGFILE} 2>&1 & + wait_for_jobs 3 done -fi + wait_for_jobs 1 +} + +rel_val_aod() +{ + local file1=${1} + local file2=${2} + local label1=${3} + local label2=${4} + local output=${OUTPUT}/AOD + local output_all=${output}/all + local output_all_bad=${output_all}_BAD + rm -r ${output} 2>/dev/null + + echo "Full RelVal to ${output_all}" | tee -a ${LOGFILE} + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${file1} -j ${file2} -o ${output_all} --labels ${label1} ${label2} 2>&1 | tee -a ${LOGFILE} + echo "Extract BAD from ${output_all} and write to ${output_all_bad}" | tee -a ${LOGFILE} + ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path ${output_all} --output ${output_all_bad} --interpretations BAD 2>&1 | tee -a ${LOGFILE} +} + +print_help() +{ + echo "Usage:" + echo "run_data_rel_val.sh [--qc ] [--aod ] [--labels ]" +} + +# Files and labels +AOD1= +AOD2= +QC1= +QC2= +LABEL1="label1" +LABEL2="label2" + +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --qc) + shift + QC1=${1} + shift + QC2=${1} + shift + ;; + --aod) + shift + AOD1=${1} + shift + AOD2=${1} + shift + ;; + --labels) + shift + LABEL1=${1} + shift + LABEL2=${1} + shift + ;; + --help|-h) + print_help + exit 0 + ;; + *) + echo "ERROR: Unknown argument ${1}" + print_help + exit 1 + ;; + esac +done + +QC_RET=0 +AOD_RET=0 + +mkdir ${OUTPUT} 2>/dev/null +echo "Do RelVal and output to ${OUTPUT}" | tee -a ${LOGFILE} + +[[ "${QC1}" != "" && "${QC2}" != "" ]] && { rel_val_qc ${QC1} ${QC2} ${LABEL1} ${LABEL2} ; QC_RET=${?} ; } || { echo "No QC RelVal" | tee -a ${LOGFILE} ; } +[[ "${AOD1}" != "" && "${AOD2}" != "" ]] && { rel_val_aod ${AOD1} ${AOD2} ${LABEL1} ${LABEL2} ; AOD_RET=${?} ; } || { echo "No AOD RelVal" | tee -a ${LOGFILE} ; } + +RET=$((QC_RET + AOD_RET)) +echo "Exit with ${RET}" | tee -a ${LOGFILE} +exit ${RET} From 7be36f9d4550fe9e039ee1e40609ab75599eb23a Mon Sep 17 00:00:00 2001 From: Markus Fasel Date: Fri, 8 Dec 2023 14:37:28 +0100 Subject: [PATCH 0269/1239] [FOCAL-24] Add trigger for particle in FOCAL acceptance --- MC/config/PWGGAJE/ini/hook_prompt_gamma_focal.ini | 4 ++++ MC/config/PWGGAJE/ini/trigger_prompt_gamma_focal.ini | 6 ++++++ MC/run/common/detector_acceptance.C | 11 ++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 MC/config/PWGGAJE/ini/hook_prompt_gamma_focal.ini create mode 100644 MC/config/PWGGAJE/ini/trigger_prompt_gamma_focal.ini diff --git a/MC/config/PWGGAJE/ini/hook_prompt_gamma_focal.ini b/MC/config/PWGGAJE/ini/hook_prompt_gamma_focal.ini new file mode 100644 index 000000000..7666aea53 --- /dev/null +++ b/MC/config/PWGGAJE/ini/hook_prompt_gamma_focal.ini @@ -0,0 +1,4 @@ +[GeneratorPythia8] +config=pythia8.cfg +hooksFileName=${O2DPG_ROOT}/MC/config/PWGGAJE/hooks/prompt_gamma_hook.C +hooksFuncName=pythia8_userhooks_promptgamma(7) \ No newline at end of file diff --git a/MC/config/PWGGAJE/ini/trigger_prompt_gamma_focal.ini b/MC/config/PWGGAJE/ini/trigger_prompt_gamma_focal.ini new file mode 100644 index 000000000..15c3cf618 --- /dev/null +++ b/MC/config/PWGGAJE/ini/trigger_prompt_gamma_focal.ini @@ -0,0 +1,6 @@ +[GeneratorPythia8] +config=pythia8.cfg + +[TriggerExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGGAJE/trigger/prompt_gamma.C +funcName=prompt_gamma(7) diff --git a/MC/run/common/detector_acceptance.C b/MC/run/common/detector_acceptance.C index 5cdd0b65a..3d6fd1d1c 100644 --- a/MC/run/common/detector_acceptance.C +++ b/MC/run/common/detector_acceptance.C @@ -18,7 +18,8 @@ enum EDetectorAcceptance_t { kAcceptance_PHSDMC, kAcceptance_EMC, kAcceptance_DMC, - kAcceptance_PHS + kAcceptance_PHS, + kAcceptance_FOC }; /// Open selection, minimum particle eta cut. @@ -122,6 +123,12 @@ bool barrel_calorimeters(Float_t phi, Float_t eta) return false; } + +/// Check if particle is in FOCAL +bool focal(Float_t phi, Float_t eta){ + return (eta > 3.4 && eta < 5.8); +} + /// \return True if particle in desired acceptance. /// /// \param acceptance : Detector acceptance to be checked. @@ -152,6 +159,8 @@ bool detector_acceptance(Int_t acceptance, Float_t phi, Float_t eta) case kAcceptance_EMCPHSDMC: return barrel_calorimeters(phi, eta); break; + case kAcceptance_FOC: + return focal(phi, eta); } return false; From a09ac5af2b434cd1c521616dda8e2d8ce2388a77 Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 11 Dec 2023 12:49:38 +0100 Subject: [PATCH 0270/1239] Seed digitizers to TF seeds for reproducible operation --- MC/bin/o2dpg_sim_workflow.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 2e1883730..03e26e02f 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -824,7 +824,11 @@ def putConfigValuesNew(listOfMainKeys=[], localCF = {}): TPCDigitask=createTask(name='tpcdigi_'+str(tf), needs=tpcdigineeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI"], cpu=NWORKERS, mem='9000') TPCDigitask['cmd'] = ('','ln -nfs ../bkg_HitsTPC.root . ;')[doembedding] - TPCDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + ' --tpc-lanes ' + str(NWORKERS) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini --early-forward-policy always ' + putConfigValuesNew(["TPCGasParam","TPCGEMParam","TPCEleParam","TPCITCorr","TPCDetParam"],localCF={"DigiParams.maxOrbitsToDigitize" : str(orbitsPerTF)}) + TPCDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + ' --tpc-lanes ' + str(NWORKERS) \ + + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini --early-forward-policy always ' \ + + putConfigValuesNew(["TPCGasParam","TPCGEMParam","TPCEleParam","TPCITCorr","TPCDetParam"], + localCF={"DigiParams.maxOrbitsToDigitize" : str(orbitsPerTF), "DigiParams.seed" : str(TFSEED)}) TPCDigitask['cmd'] += (' --tpc-chunked-writer','')[args.no_tpc_digitchunking] TPCDigitask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] # we add any other extra command line options (power user customization) with an environment variable @@ -838,7 +842,9 @@ def putConfigValuesNew(listOfMainKeys=[], localCF = {}): TRDDigitask=createTask(name='trddigi_'+str(tf), needs=trddigineeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI"], cpu=NWORKERS, mem='8000') TRDDigitask['cmd'] = ('','ln -nfs ../bkg_HitsTRD.root . ;')[doembedding] - TRDDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --onlyDet TRD --interactionRate ' + str(INTRATE) + putConfigValuesNew(localCF={"TRDSimParams.digithreads" : NWORKERS}) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' + TRDDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + + ' --onlyDet TRD --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' \ + + putConfigValuesNew(localCF={"TRDSimParams.digithreads" : NWORKERS, "DigiParams.seed" : str(TFSEED)}) TRDDigitask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] if isActive("TRD"): workflow['stages'].append(TRDDigitask) @@ -848,7 +854,10 @@ def createRestDigiTask(name, det='ALLSMALLER'): tneeds =[ContextTask['name']] if includeQED == True: tneeds += [QED_task['name']] - commondigicmd = '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' + putConfigValuesNew(["MFTAlpideParam", "ITSAlpideParam", "ITSDigitizerParam"]) + QEDdigiargs + commondigicmd = '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + + ' --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' \ + + putConfigValuesNew(["MFTAlpideParam", "ITSAlpideParam", "ITSDigitizerParam"], + localCF={"DigiParams.seed" : str(TFSEED), "MCHDigitizer.seed" : str(TFSEED)}) + QEDdigiargs if det=='ALLSMALLER': # here we combine all smaller digits in one DPL workflow if usebkgcache: @@ -902,7 +911,10 @@ def createRestDigiTask(name, det='ALLSMALLER'): t = createTask(name="ft0fv0emcctp_digi_" + str(tf), needs=tneeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') t['cmd'] = ('','ln -nfs ../bkg_HitsFT0.root . ; ln -nfs ../bkg_HitsFV0.root . ;')[doembedding] - t['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption + ' --onlyDet FT0,FV0,EMC,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' + putConfigValuesNew() + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs + t['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + + ' --onlyDet FT0,FV0,EMC,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) \ + + ' --disable-write-ini' + putConfigValuesNew(localCF={"DigiParams.seed" : str(TFSEED)}) \ + + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs workflow['stages'].append(t) det_to_digitask["FT0"]=t det_to_digitask["FV0"]=t From 817e7bfe85bb9227251a09a1fc08a98ec3942bb4 Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 12 Dec 2023 08:26:40 +0100 Subject: [PATCH 0271/1239] expand error detection grep --- GRID/utils/extractFailureMessage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/GRID/utils/extractFailureMessage.sh b/GRID/utils/extractFailureMessage.sh index 548666e9a..dd3d9a07d 100755 --- a/GRID/utils/extractFailureMessage.sh +++ b/GRID/utils/extractFailureMessage.sh @@ -24,6 +24,7 @@ pattern="-e \"\<[Ee]xception\" \ -e \"\]\[FATAL\]\" \ -e \"\*\*\* Program crashed\" \ -e \"\*\*\* Fatal Exception\" \ + -e \"was killed abnormally\" \ -e \"\*\*\* Error in\"" # <--- LIBC fatal error messages CMD="grep -m 1 -B2 -A10 ${pattern} $filetocheck" From d90682e0db6210182196b61e79336739ebcf768f Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 12 Dec 2023 08:58:01 +0100 Subject: [PATCH 0272/1239] make script platform independent --- GRID/utils/extractErroredLogFiles.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/GRID/utils/extractErroredLogFiles.sh b/GRID/utils/extractErroredLogFiles.sh index a8d579585..81c21839d 100755 --- a/GRID/utils/extractErroredLogFiles.sh +++ b/GRID/utils/extractErroredLogFiles.sh @@ -1,15 +1,19 @@ +#!/usr/bin/env bash + # We determine the O2DPG task that failed (as listed in stdout) and extract the relevant log automatically # Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs -alias mytar=tar +mytar () { + tar $@ +} if [[ $(uname) == "Darwin" ]]; then echo "Running on macOS. This needs gnu-tar" - alias mytar=gtar + $(which gtar) + mytar () { + gtar $@ + } fi -# show aliases -alias - errored_tasks="" find ./ -name "stdout*" -exec grep -H "failed.*retry" {} ';' | sed 's/ failed.*//' | tr ":" " " | while IFS= read -r line; do stdoutpath=$(echo "$line" | awk '{print $1}') # Extracting the first column From f532f4cc6c6383a33503396cc69521a6112d5ddf Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 12 Dec 2023 10:38:47 +0100 Subject: [PATCH 0273/1239] Keep TOF output if explicitly requested, not just in CPass --- DATA/production/configurations/asyncReco/async_pass.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 5cf53e89c..6e4312a49 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -242,7 +242,7 @@ if [[ ! -z "$ALIEN_JDL_DDSHMSIZE" ]]; then export DDSHMSIZE=$ALIEN_JDL_DDSHMSIZE SETTING_ROOT_OUTPUT="ENABLE_ROOT_OUTPUT_o2_mch_reco_workflow= ENABLE_ROOT_OUTPUT_o2_muon_tracks_matcher_workflow= ENABLE_ROOT_OUTPUT_o2_aod_producer_workflow= ENABLE_ROOT_OUTPUT_o2_qc= " -if [[ -n $ALIEN_JDL_LPMCPASSMODE ]] && [[ $ALIEN_JDL_LPMCPASSMODE != "-1" ]]; then +if ([[ -n $ALIEN_JDL_LPMCPASSMODE ]] && [[ $ALIEN_JDL_LPMCPASSMODE != "-1" ]]) || [[ $ALIEN_JDL_KEEPTOFMATCHOUTPUT == "1" ]]; then SETTING_ROOT_OUTPUT+="ENABLE_ROOT_OUTPUT_o2_tof_matcher_workflow= " fi if [[ $ALIEN_JDL_DOEMCCALIB == "1" ]]; then From dad842a95af89b8e424d47ef6dd40b6f589b88df Mon Sep 17 00:00:00 2001 From: swenzel Date: Tue, 12 Dec 2023 15:29:32 +0100 Subject: [PATCH 0274/1239] Better construction of configKey values for transport simulation So far, transport sim takes configKey values solely from the command line. But we might want to apply default configs directly in the script. This is achieved with this commit. --- MC/bin/o2dpg_sim_config.py | 38 ++++++++++++++++++++++++++++++++++++ MC/bin/o2dpg_sim_workflow.py | 23 +++++++++------------- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index 279f73b36..01692d8ff 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -86,3 +86,41 @@ def add(cfg, flatconfig): add(config, {"ITSVertexerParam.lowMultBeamDistCut": "0."}) return config + + +def create_geant_config(args, externalConfigString): + # creates generic transport simulation config key values + # based on arguments args (run number, energy, ...) originally passed + # to o2dpg_sim_workflow.py + config = {} + def add(cfg, flatconfig): + for entry in flatconfig: + mk = entry.split(".")[0] + sk = entry.split(".")[1] + d = cfg.get(mk,{}) + d[sk] = flatconfig[entry] + cfg[mk] = d + + # ----- add default settings ----- + + add(config, {"MFTBase.buildAlignment" : "true"}) + + # ----- apply external overwrites from command line ------- + for keyval in externalConfigString.split(";"): + if len(keyval) > 0: + key, val = keyval.split("=") + add(config, {key : val}) + + return config + +def constructConfigKeyArg(config): + # flattens dictionary constructed in create_geant_config + # and constructs the --configKeyValues options for simulation + if len(config) == 0: + return '' + arg = '--configKeyValues "' + for mainkey in config: + for subkey in config[mainkey]: + arg = arg + mainkey + '.' + subkey + '=' + config[mainkey][subkey] + ';' + arg = arg + '"' + return arg diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 03e26e02f..70d81c64f 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -36,7 +36,7 @@ from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector from o2dpg_qc_finalization_workflow import include_all_QC_finalization -from o2dpg_sim_config import create_sim_config +from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg parser = argparse.ArgumentParser(description='Create an ALICE (Run3) MC simulation workflow') @@ -405,10 +405,6 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): print('o2dpg_sim_workflow: Error! bkg ECM or Beam Energy not set!!!') exit(1) - CONFKEYBKG='' - if args.confKeyBkg!= '': - CONFKEYBKG=' --configKeyValues "' + args.confKeyBkg + '"' - # Background PYTHIA configuration BKG_CONFIG_task=createTask(name='genbkgconf') BKG_CONFIG_task['cmd'] = 'echo "placeholder / dummy task"' @@ -425,7 +421,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): --process='+str(PROCESSBKG) # if we configure pythia8 here --> we also need to adjust the configuration # TODO: we need a proper config container/manager so as to combine these local configs with external configs etc. - CONFKEYBKG='--configKeyValues "GeneratorPythia8.config=pythia8bkg.cfg;' + args.confKeyBkg + '"' + args.confKeyBkg = 'GeneratorPythia8.config=pythia8bkg.cfg;' + args.confKeyBkg workflow['stages'].append(BKG_CONFIG_task) @@ -434,6 +430,9 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): if args.iniBkg!= '': INIBKG=' --configFile ' + args.iniBkg + # determine final configKey values for background transport + CONFKEYBKG = constructConfigKeyArg(create_geant_config(args, args.confKeyBkg)) + BKGtask=createTask(name='bkgsim', lab=["GEANT"], needs=[BKG_CONFIG_task['name'], GRP_TASK['name']], cpu=NWORKERS ) BKGtask['cmd']='${O2_ROOT}/bin/o2-sim -e ' + SIMENGINE + ' -j ' + str(NWORKERS) + ' -n ' + str(NBKGEVENTS) \ + ' -g ' + str(GENBKG) + ' ' + str(MODULES) + ' -o bkg ' + str(INIBKG) \ @@ -527,9 +526,6 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): INIFILE='' if args.ini!= '': INIFILE=' --configFile ' + args.ini - CONFKEY='' - if args.confKey!= '': - CONFKEY=' --configKeyValues "' + args.confKey + '"' PROCESS=args.proc TRIGGER='' if args.trigger != '': @@ -645,16 +641,12 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): SGN_CONFIG_task['cmd'] = SGN_CONFIG_task['cmd'] + ' --weightPow=' + str(WEIGHTPOW) # if we configure pythia8 here --> we also need to adjust the configuration # TODO: we need a proper config container/manager so as to combine these local configs with external configs etc. - CONFKEY='--configKeyValues "GeneratorPythia8.config=pythia8.cfg'+';'+args.confKey+'"' + args.confKey = args.confKey + ";GeneratorPythia8.config=pythia8.cfg" # elif GENERATOR == 'extgen': what do we do if generator is not pythia8? # NOTE: Generator setup might be handled in a different file or different files (one per # possible generator) - #if CONFKEY=='': - # print('o2dpg_sim_workflow: Error! configuration file not provided') - # exit(1) - workflow['stages'].append(SGN_CONFIG_task) # ----------------- @@ -664,6 +656,9 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): if (args.pregenCollContext == True): signalneeds.append(PreCollContextTask['name']) + # determine final configKey args for signal transport + CONFKEY = constructConfigKeyArg(create_geant_config(args, args.confKey)) + # add embedIntoFile only if embeddPattern does contain a '@' embeddinto= "--embedIntoFile ../bkg_MCHeader.root" if (doembedding & ("@" in args.embeddPattern)) else "" #embeddinto= "--embedIntoFile ../bkg_MCHeader.root" if doembedding else "" From c6db4fe03814c794ebf27c04f4c4290462856abb Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Tue, 12 Dec 2023 16:18:10 +0100 Subject: [PATCH 0275/1239] python script to binary compare 2 ROOT files (same structure) --- UTILS/cmpROOTFiles.py | 135 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100755 UTILS/cmpROOTFiles.py diff --git a/UTILS/cmpROOTFiles.py b/UTILS/cmpROOTFiles.py new file mode 100755 index 000000000..78a3d5ee1 --- /dev/null +++ b/UTILS/cmpROOTFiles.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import ROOT +import argparse + +parser = argparse.ArgumentParser(description='Check if 2 ROOT files are binary compatible', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument('-f1','--file1', help='First ROOT TFile', required=True) +parser.add_argument('-f2','--file2', help='Second ROOT TFile', required=True) +args = parser.parse_args() + +def get_total_branch_list(tree): + branches = [] + + # Function to recursively get branches + def get_branches_recursive(branch): + branches.append(branch) + sub_branches = branch.GetListOfBranches() + + if sub_branches: + for sub_branch in sub_branches: + get_branches_recursive(sub_branch) + + # Get top-level branches of the tree + top_level_branches = tree.GetListOfBranches() + + # Traverse recursively through branches + for branch in top_level_branches: + get_branches_recursive(branch) + + return branches + + +def compare_branches(obj1, obj2): + # Check if the object classes match + if obj1.IsA() != obj2.IsA(): + print("Type doesn't match") + return False + + # Check if the byte content is the same + if obj1.GetTitle() != obj2.GetTitle(): + print ("Title doesn't match") + return False + + # Convert objects to TBuffer to compare their byte content + #buffer1 = ROOT.TBuffer(ROOT.TBuffer.EMode.kWrite, 10000) + #buffer2 = ROOT.TBuffer(ROOT.TBuffer.EMode.kWrite, 10000) + + #obj1.Streamer(buffer1) + #obj2.Streamer(buffer2) + # checking branch + print ("Checking branch " + obj1.GetTitle()) + if obj1.GetTotBytes() != obj2.GetTotBytes(): + print ("Bytecount different") + return False + + +# compare 2 TTree objects +def compare_trees(tree1, tree2): + branches1 = get_total_branch_list(tree1) + branches2 = get_total_branch_list(tree2) + + # branch count needs to be same + if len(branches1) != len(branches2): + return False + + # we do not impose same branch order so we build 2 hashsets containing tuples + # of (branchname, totalsize) + + set1 = set() + for br in branches1: + # Print key name and class name + #print("Key: ", br.GetName()) + #print("Class: ", br.ClassName()) + #print("BC: ", str(br.GetTotalSize())) + #print("---------------") + + totals = 0 + for entry in range(br.GetEntries()): + totals = totals + br.GetEntry(entry) + set1.add((br.GetName(), totals, br.GetEntries())) + + set2 = set() + for br in branches2: + totals = 0 + for entry in range(br.GetEntries()): + totals = totals + br.GetEntry(entry) + set2.add((br.GetName(), totals, br.GetEntries())) + + inters = set1.intersection(set2) + #print (inters) + symdiff = (set1.symmetric_difference(set2)) + if (len(symdiff) > 0): + print (symdiff) + return len(symdiff) == 0 + +def compare_root_files(file1, file2): + # Open the ROOT files + tfile1 = ROOT.TFile.Open(file1) + tfile2 = ROOT.TFile.Open(file2) + + # Get the list of keys (TKeys) in the ROOT files + keys1 = tfile1.GetListOfKeys() + keys2 = tfile2.GetListOfKeys() + + # Check if the number of keys is the same + if keys1.GetEntries() != keys2.GetEntries(): + return False + + # Iterate through the keys and compare byte content + # check keys + success = True + for key1, key2 in zip(keys1, keys2): + obj1 = key1.ReadObj() + obj2 = key2.ReadObj() + + isTree1 = isinstance(obj1, ROOT.TTree) + isTree2 = isinstance(obj2, ROOT.TTree) + if isTree1 != isTree2: + success = False + + elif isTree1 and isTree2: + success = success and compare_trees(obj1, obj2) + + # Close the files + tfile1.Close() + tfile2.Close() + + return success + +result = compare_root_files(args.file1, args.file2) +if result: + print("Byte content of the two ROOT files is the same.", args.file1, args.file2) +else: + print("Byte content of the two ROOT files is different.", args.file1, args.file2) From f9fa0c44a606971afc14b116f581f20b614f0dea Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Wed, 13 Dec 2023 11:14:38 +0100 Subject: [PATCH 0276/1239] [RelVal] Update README.md --- RelVal/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/RelVal/README.md b/RelVal/README.md index 98ac4af0e..b2d1c7d2a 100644 --- a/RelVal/README.md +++ b/RelVal/README.md @@ -14,6 +14,10 @@ The convention is, that only those objects that have the exact same path are com At the end of this README are some examples for QC RelVal. +## Quick start + +To jump right in, please check out [this](#run-for-qc) + ## Definitions ### Metric @@ -127,7 +131,10 @@ ${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${MC_PRODUCTION}/QC/ ## Run for QC This is a simple guide to run RelVal for QC. -Here is also a [working example](run/run_data_rel_val.sh). +Here is also a [working example](run/run_data_rel_val.sh), run it with +```bash +${O2DPG_ROOT}/RelVal/run/run_data_rel_val.sh [--qc QC1.root QC2.root ] [--aod AOD1.root AOD2.root] [ --labels LABEL1 LABEL2] +``` ### If you are interested in all QC plots To have everything and to use this as a starting point for deeper inspections, first run From d0604264c3b9758d2cae7fbd5be1a2d350723c17 Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Wed, 13 Dec 2023 10:02:38 +0100 Subject: [PATCH 0277/1239] Updated MC file; haven't been updated for a year so there were some histos missing in QC output --- MC/config/QC/json/tpc-qc-standard-direct.json | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/MC/config/QC/json/tpc-qc-standard-direct.json b/MC/config/QC/json/tpc-qc-standard-direct.json index 6f560a5ce..502da9d25 100644 --- a/MC/config/QC/json/tpc-qc-standard-direct.json +++ b/MC/config/QC/json/tpc-qc-standard-direct.json @@ -3,11 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable", - "maxObjectSize": "20971520" + "host": "ccdb-test.cern.ch:8080" }, "Activity": { "number": "42", @@ -37,7 +33,6 @@ "moduleName": "QcTPC", "detectorName": "TPC", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "inputClus:TPC/CLUSTERNATIVE" @@ -58,7 +53,6 @@ "moduleName": "QcTPC", "detectorName": "TPC", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query" : "inputTracks:TPC/TRACKS/0" @@ -88,7 +82,19 @@ "taskParameters": { "cutAbsEta": "1.", "cutMinNCluster": "60", - "cutMindEdxTot": "20." + "cutMindEdxTot": "20.", + "samplingFractionDCAr": "0.1", + "cutPtForDCAr": "1.5" + }, + "grpGeomRequest" : { + "geomRequest": "None", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "true", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" } } } From d8a666cd0939159b55f615a4de75815741e10d2c Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Wed, 13 Dec 2023 16:37:31 +0100 Subject: [PATCH 0278/1239] Fixed conditionDB --- MC/config/QC/json/tpc-qc-standard-direct.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/QC/json/tpc-qc-standard-direct.json b/MC/config/QC/json/tpc-qc-standard-direct.json index 502da9d25..156676ad1 100644 --- a/MC/config/QC/json/tpc-qc-standard-direct.json +++ b/MC/config/QC/json/tpc-qc-standard-direct.json @@ -23,7 +23,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch:8080" } }, "tasks": { From e6138b59ef68d0558d3b4b8e33d503fc808546ea Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Thu, 14 Dec 2023 11:49:36 +0100 Subject: [PATCH 0279/1239] Env var starting with SPLIT are ignored by JDL parser --- DATA/production/configurations/asyncReco/async_pass.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 6e4312a49..71fdb4f5c 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -444,7 +444,7 @@ fi # reco and matching # print workflow -if [[ $ALIEN_JDL_SPLITWF != "1" ]]; then +if [[ $ALIEN_JDL_SSPLITWF != "1" ]]; then env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list > workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then @@ -476,7 +476,7 @@ else echo "We will run the workflow in SPLIT mode!" WORKFLOW_PARAMETERS_START=$WORKFLOW_PARAMETERS - if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 1 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 1 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" == "all" ]]; then + if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 1 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 1 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" == "all" ]]; then # 1. TPC decoding + reco echo "Step 1) Decoding and reconstructing TPC+CTP" echo "Step 1) Decoding and reconstructing TPC+CTP" > workflowconfig.log @@ -513,7 +513,7 @@ else fi fi - if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 2 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 2 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" == "all" ]]; then + if ([[ -z "$ALIEN_JDL_STARTSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 2 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 2 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" == "all" ]]; then # 2. the other detectors decoding + reco WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START echo "Step 2) Decoding and reconstructing ALL-TPC" @@ -574,7 +574,7 @@ else fi fi - if ([[ -z "$ALIEN_JDL_SPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SPLITSTEP" ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SPLITSTEP" -eq "all" ]]; then + if ([[ -z "$ALIEN_JDL_SSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq "all" ]]; then # 3. matching, QC, calib, AOD WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START echo "Step 3) matching, QC, calib, AOD" From ccbde7a5064d93452a33395c0f27fc42122a263c Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Thu, 14 Dec 2023 11:59:20 +0100 Subject: [PATCH 0280/1239] removing port in json for alice-ccdb.cern.ch --- MC/config/QC/json/tpc-qc-standard-direct.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/QC/json/tpc-qc-standard-direct.json b/MC/config/QC/json/tpc-qc-standard-direct.json index 156676ad1..c3d3b478e 100644 --- a/MC/config/QC/json/tpc-qc-standard-direct.json +++ b/MC/config/QC/json/tpc-qc-standard-direct.json @@ -23,7 +23,7 @@ "url": "" }, "conditionDB": { - "url": "alice-ccdb.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { From 9c58ded9fd0578f82c1516cc7af68e66f4917eff Mon Sep 17 00:00:00 2001 From: mcoquet642 Date: Fri, 8 Dec 2023 16:48:56 +0100 Subject: [PATCH 0281/1239] Use corrected MFT geometry and apply non-ideal alignment --- MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh index 3289e6e15..577032fa6 100755 --- a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh +++ b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh @@ -94,8 +94,8 @@ baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${A # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ - -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ - --include-local-qc --include-analysis --mft-reco-full" + -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41;MFTBase.buildAlignment=true\" \ + --include-local-qc --include-analysis" remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" @@ -117,7 +117,7 @@ echo "TIMESTAMP IS ${TIMESTAMP}" export ALICEO2_CCDB_LOCALCACHE=$PWD/.ccdb [ ! -d .ccdb ] && mkdir .ccdb -CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /FT0/Calibration/ChannelTimeOffset /FV0/Calibration/ChannelTimeOffset /GLO/GRP/BunchFilling" +CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /MFT/Calib/Align /FT0/Calibration/ChannelTimeOffset /FV0/Calibration/ChannelTimeOffset /GLO/GRP/BunchFilling" ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS} -d .ccdb --timestamp ${TIMESTAMP} if [ ! "$?" == "0" ]; then @@ -125,8 +125,8 @@ if [ ! "$?" == "0" ]; then exit 1 fi -# -- Create aligned geometry using ITS and MFT ideal alignments to avoid overlaps in geant -CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align MFT/Calib/Align" +# -- Create aligned geometry using ITS ideal alignment to avoid overlaps in geant +CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align" TIMESTAMP_IDEAL_MC=1 ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS_IDEAL_MC} -d .ccdb --timestamp ${TIMESTAMP_IDEAL_MC} if [ ! "$?" == "0" ]; then From 382aaa0bd352e0986b293fdaaaa379afb8bd3900 Mon Sep 17 00:00:00 2001 From: mcoquet642 Date: Fri, 8 Dec 2023 17:01:52 +0100 Subject: [PATCH 0282/1239] Removing deprecated mft reco parameters --- MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh | 2 +- MC/run/PWGDQ/runBeautyToJpsi_fwdy_PbPb.sh | 2 +- MC/run/PWGDQ/runBeautyToJpsi_fwdy_pp.sh | 2 +- MC/run/PWGDQ/runBeautyToJpsi_midy_pp.sh | 2 +- MC/run/PWGDQ/runBeautyToMuons_fwd_pp.sh | 2 +- MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh | 2 +- MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh | 2 +- MC/run/PWGDQ/runBeautyToPsi_fwdy_PbPb.sh | 2 +- MC/run/PWGDQ/runBeautyToPsi_fwdy_pp.sh | 2 +- MC/run/PWGDQ/runBeautyToPsi_midy_pp.sh | 2 +- MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh | 2 +- MC/run/PWGDQ/runCharmToMuons_fwd_pp.sh | 2 +- MC/run/PWGDQ/runFwdMuBoxGen_PbPb.sh | 2 +- MC/run/PWGDQ/runFwdMuBoxGen_pp.sh | 2 +- MC/run/PWGDQ/runPromptCharmonia_fwdy_PbPb.sh | 2 +- MC/run/PWGDQ/runPromptCharmonia_fwdy_pp.sh | 2 +- MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment.sh | 2 +- .../PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh | 2 +- MC/run/PWGDQ/runPromptCharmonia_midy_pp.sh | 2 +- MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh | 2 +- MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh | 2 +- MC/run/PWGDQ/runPromptJpsi_midy_pp.sh | 2 +- MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh | 2 +- 23 files changed, 23 insertions(+), 23 deletions(-) diff --git a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh index 01563e70b..52bae7c40 100755 --- a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh +++ b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh @@ -105,7 +105,7 @@ baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${A remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ - --include-local-qc --include-analysis --mft-reco-full" + --include-local-qc --include-analysis" remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" diff --git a/MC/run/PWGDQ/runBeautyToJpsi_fwdy_PbPb.sh b/MC/run/PWGDQ/runBeautyToJpsi_fwdy_PbPb.sh index b2b0edde4..5db05ff3d 100644 --- a/MC/run/PWGDQ/runBeautyToJpsi_fwdy_PbPb.sh +++ b/MC/run/PWGDQ/runBeautyToJpsi_fwdy_PbPb.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 5020 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_fwdy.ini \ - -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata + -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToJpsi_fwdy_pp.sh b/MC/run/PWGDQ/runBeautyToJpsi_fwdy_pp.sh index 8d6a8dca7..25f2249e6 100755 --- a/MC/run/PWGDQ/runBeautyToJpsi_fwdy_pp.sh +++ b/MC/run/PWGDQ/runBeautyToJpsi_fwdy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_fwdy.ini \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full --fwdmatching-4-param --fwdmatching-cut-4-param + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --fwdmatching-4-param --fwdmatching-cut-4-param # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToJpsi_midy_pp.sh b/MC/run/PWGDQ/runBeautyToJpsi_midy_pp.sh index ffc534a4d..1a09db46a 100755 --- a/MC/run/PWGDQ/runBeautyToJpsi_midy_pp.sh +++ b/MC/run/PWGDQ/runBeautyToJpsi_midy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_midy.ini \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToMuons_fwd_pp.sh b/MC/run/PWGDQ/runBeautyToMuons_fwd_pp.sh index 92118177f..b4fbe123a 100644 --- a/MC/run/PWGDQ/runBeautyToMuons_fwd_pp.sh +++ b/MC/run/PWGDQ/runBeautyToMuons_fwd_pp.sh @@ -20,7 +20,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "MCH MFT MID ITS" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbarToMuonsSemileptonic_fwdy.ini \ - -genBkg pythia8 -procBkg cdiff -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full \ + -genBkg pythia8 -procBkg cdiff -colBkg pp --embedding -nb ${NBKGEVENTS} \ -confKeyBkg "Diamond.width[2]=6" -interactionRate 2000 --mft-assessment-full --fwdmatching-assessment-full # run workflow diff --git a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh index 90f4fac9a..642bed9df 100755 --- a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh +++ b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_fwdy_pp_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_fwdy_triggerGap.ini --mft-assessment-full --fwdmatching-assessment-full # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh index 4b0f86376..2a2edcc42 100755 --- a/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh +++ b/MC/run/PWGDQ/runBeautyToPsiAndJpsi_midy_pp_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini --mft-reco-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_PsiAndJpsi_midy_triggerGap.ini # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToPsi_fwdy_PbPb.sh b/MC/run/PWGDQ/runBeautyToPsi_fwdy_PbPb.sh index 3ef8fda82..e791a1502 100755 --- a/MC/run/PWGDQ/runBeautyToPsi_fwdy_PbPb.sh +++ b/MC/run/PWGDQ/runBeautyToPsi_fwdy_PbPb.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 5020 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_Psi2S_fwdy.ini \ - -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToPsi_fwdy_pp.sh b/MC/run/PWGDQ/runBeautyToPsi_fwdy_pp.sh index c55192b0e..f1f18ae86 100755 --- a/MC/run/PWGDQ/runBeautyToPsi_fwdy_pp.sh +++ b/MC/run/PWGDQ/runBeautyToPsi_fwdy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_Psi2S_fwdy.ini \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBeautyToPsi_midy_pp.sh b/MC/run/PWGDQ/runBeautyToPsi_midy_pp.sh index 01bba73f4..001abad3b 100755 --- a/MC/run/PWGDQ/runBeautyToPsi_midy_pp.sh +++ b/MC/run/PWGDQ/runBeautyToPsi_midy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbar_Psi2S_midy.ini \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh index 0e74920d8..04402b17b 100755 --- a/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh +++ b/MC/run/PWGDQ/runBplusToJpsi_midy_pp_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini --mft-reco-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbarToBplus_midy_triggerGap.ini # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runCharmToMuons_fwd_pp.sh b/MC/run/PWGDQ/runCharmToMuons_fwd_pp.sh index 327298964..565a01aca 100644 --- a/MC/run/PWGDQ/runCharmToMuons_fwd_pp.sh +++ b/MC/run/PWGDQ/runCharmToMuons_fwd_pp.sh @@ -20,7 +20,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "MCH MFT MID ITS" \ -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_ccbarToMuonsSemileptonic_fwdy.ini \ - -genBkg pythia8 -procBkg cdiff -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full \ + -genBkg pythia8 -procBkg cdiff -colBkg pp --embedding -nb ${NBKGEVENTS} \ -confKeyBkg "Diamond.width[2]=6" -interactionRate 2000 --mft-assessment-full --fwdmatching-assessment-full # run workflow diff --git a/MC/run/PWGDQ/runFwdMuBoxGen_PbPb.sh b/MC/run/PWGDQ/runFwdMuBoxGen_PbPb.sh index a5e34d1e9..7fd5435ee 100755 --- a/MC/run/PWGDQ/runFwdMuBoxGen_PbPb.sh +++ b/MC/run/PWGDQ/runFwdMuBoxGen_PbPb.sh @@ -18,7 +18,7 @@ NBOXMUONS=${NBOXMUONS:-2} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 5020 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorBoxFwd.C;GeneratorExternal.funcName=fwdMuBoxGen()" \ - -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata + -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata # run workflow (MFT-related tasks) NMUONS=$NBOXMUONS ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt "(mft.*)" diff --git a/MC/run/PWGDQ/runFwdMuBoxGen_pp.sh b/MC/run/PWGDQ/runFwdMuBoxGen_pp.sh index 1d54c8e6e..6d154e06e 100755 --- a/MC/run/PWGDQ/runFwdMuBoxGen_pp.sh +++ b/MC/run/PWGDQ/runFwdMuBoxGen_pp.sh @@ -18,7 +18,7 @@ NBOXMUONS=${NBOXMUONS:-2} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorBoxFwd.C;GeneratorExternal.funcName=fwdMuBoxGen()" \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata # run workflow (MFT-related tasks) NMUONS=$NBOXMUONS ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt "(mft.*)" diff --git a/MC/run/PWGDQ/runPromptCharmonia_fwdy_PbPb.sh b/MC/run/PWGDQ/runPromptCharmonia_fwdy_PbPb.sh index 725cbaa6b..d305c150e 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_fwdy_PbPb.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_fwdy_PbPb.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 5020 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C;GeneratorExternal.funcName=GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV()" \ - -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata + -genBkg pythia8 -procBkg "heavy_ion" -colBkg PbPb --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full --fwdmatching-save-trainingdata # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp.sh b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp.sh index 0b4465e43..dc0c28fae 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp.sh @@ -17,7 +17,7 @@ TARGETTASK=${TARGETTASK:+-tt ${TARGETTASK}} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV()" \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full --fwdmatching-4-param --fwdmatching-cut-4-param + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --fwdmatching-4-param --fwdmatching-cut-4-param # run workflow diff --git a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment.sh b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment.sh index ec9cf2ccb..e4f80946e 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV()" \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-assessment-full --fwdmatching-assessment-full # run workflow diff --git a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh index fc637c33d..b64e13ee3 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_fwdy_pp_assessment_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaFwdy_TriggerGap.ini --mft-assessment-full --fwdmatching-assessment-full # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptCharmonia_midy_pp.sh b/MC/run/PWGDQ/runPromptCharmonia_midy_pp.sh index 4538eed9a..30aa3a865 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_midy_pp.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_midy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV()" \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} # run workflow diff --git a/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh b/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh index 0636e7eb6..d4ad2179a 100755 --- a/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh +++ b/MC/run/PWGDQ/runPromptCharmonia_midy_pp_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini --mft-reco-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptCharmoniaMidy_TriggerGap.ini # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh index d1f03343f..5273b9398 100755 --- a/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh +++ b/MC/run/PWGDQ/runPromptJpsi_fwdy_pp_assessment_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptJpsiFwdy_TriggerGap.ini --mft-assessment-full --fwdmatching-assessment-full # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptJpsi_midy_pp.sh b/MC/run/PWGDQ/runPromptJpsi_midy_pp.sh index 311d87c70..eed212575 100755 --- a/MC/run/PWGDQ/runPromptJpsi_midy_pp.sh +++ b/MC/run/PWGDQ/runPromptJpsi_midy_pp.sh @@ -16,7 +16,7 @@ NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 900 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ -confKey "GeneratorExternal.fileName=${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV()" \ - -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} --mft-reco-full + -genBkg pythia8 -procBkg inel -colBkg pp --embedding -nb ${NBKGEVENTS} # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json diff --git a/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh b/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh index 406152d24..26f6cc930 100755 --- a/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh +++ b/MC/run/PWGDQ/runPromptPsi2S_fwdy_pp_assessment_triggerGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} NTIMEFRAMES=${NTIMEFRAMES:-1} ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "--skipModules ZDC" \ - -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini --mft-reco-full --mft-assessment-full --fwdmatching-assessment-full + -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/Generator_InjectedPromptPsi2SFwdy_TriggerGap.ini --mft-assessment-full --fwdmatching-assessment-full # run workflow ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json From ada221eba745747a28c00b52116e7e889f310137 Mon Sep 17 00:00:00 2001 From: mcoquet642 Date: Fri, 8 Dec 2023 17:04:31 +0100 Subject: [PATCH 0283/1239] Same modifications on older script --- MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh index 52bae7c40..1dba0302d 100755 --- a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh +++ b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh @@ -104,7 +104,7 @@ baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${A # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ - -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ + -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41;MFTBase.buildAlignment=true\" \ --include-local-qc --include-analysis" remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" @@ -126,7 +126,7 @@ echo "TIMESTAMP IS ${TIMESTAMP}" export ALICEO2_CCDB_LOCALCACHE=$PWD/.ccdb [ ! -d .ccdb ] && mkdir .ccdb -CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /FT0/Calibration/ChannelTimeOffset /FV0/Calibration/ChannelTimeOffset" +CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /MFT/Calib/Align /FT0/Calibration/ChannelTimeOffset /FV0/Calibration/ChannelTimeOffset" ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS} -d .ccdb --timestamp ${TIMESTAMP} if [ ! "$?" == "0" ]; then @@ -134,8 +134,8 @@ if [ ! "$?" == "0" ]; then exit 1 fi -# -- Create aligned geometry using ITS and MFT ideal alignments to avoid overlaps in geant -CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align MFT/Calib/Align" +# -- Create aligned geometry using ITS ideal alignment to avoid overlaps in geant +CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align" TIMESTAMP_IDEAL_MC=1 ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS_IDEAL_MC} -d .ccdb --timestamp ${TIMESTAMP_IDEAL_MC} if [ ! "$?" == "0" ]; then From c436c653c2df3da6d308c7eccec9def47f4fe34f Mon Sep 17 00:00:00 2001 From: mcoquet642 Date: Tue, 12 Dec 2023 19:53:53 +0100 Subject: [PATCH 0284/1239] REmoving corrected geometry option --- MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh | 2 +- MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh index 1dba0302d..db58f773a 100755 --- a/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh +++ b/MC/run/ANCHOR/2021/OCT/pass4/anchorMC.sh @@ -104,7 +104,7 @@ baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${A # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ - -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41;MFTBase.buildAlignment=true\" \ + -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ --include-local-qc --include-analysis" remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" diff --git a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh index 577032fa6..c75b8f581 100755 --- a/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh +++ b/MC/run/ANCHOR/2022/JUN/pass1/anchorMC.sh @@ -94,7 +94,7 @@ baseargs="-tf ${NTIMEFRAMES} --split-id ${ALIEN_JDL_SPLITID:-1} --prod-split ${A # THIS NEEDS TO COME FROM OUTSIDE remainingargs="-gen pythia8 -proc cdiff -ns ${NSIGEVENTS} \ -interactionRate ${INTERACTIONRATE} \ - -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41;MFTBase.buildAlignment=true\" \ + -confKey \"Diamond.width[2]=6.0;Diamond.width[0]=0.01;Diamond.width[1]=0.01;Diamond.position[0]=0.0;Diamond.position[1]=-0.035;Diamond.position[2]=0.41\" \ --include-local-qc --include-analysis" remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" From 174c9b86849ad9a08807358ab06653de5849eb8d Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 15 Dec 2023 09:29:50 +0100 Subject: [PATCH 0285/1239] Update failure message pattern --- GRID/utils/extractFailureMessage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/GRID/utils/extractFailureMessage.sh b/GRID/utils/extractFailureMessage.sh index dd3d9a07d..4c8c391e5 100755 --- a/GRID/utils/extractFailureMessage.sh +++ b/GRID/utils/extractFailureMessage.sh @@ -25,6 +25,7 @@ pattern="-e \"\<[Ee]xception\" \ -e \"\*\*\* Program crashed\" \ -e \"\*\*\* Fatal Exception\" \ -e \"was killed abnormally\" \ + -e \"line.*Killed\" \ -e \"\*\*\* Error in\"" # <--- LIBC fatal error messages CMD="grep -m 1 -B2 -A10 ${pattern} $filetocheck" From f0aab02c9598e75ffbad34a7e8ac77fa0275ea3d Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Thu, 14 Dec 2023 14:00:59 +0100 Subject: [PATCH 0286/1239] Moving window --- DATA/production/qc-async/tpc.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index a715c29e7..6afa9aacc 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -30,7 +30,7 @@ "className": "o2::quality_control_modules::tpc::Clusters", "moduleName": "QcTPC", "detectorName": "TPC", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "300", "dataSource": { "type": "dataSamplingPolicy", "name": "tpc-clusters" @@ -51,7 +51,8 @@ "className": "o2::quality_control_modules::tpc::PID", "moduleName": "QcTPC", "detectorName": "TPC", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "300", + "movingWindows" : [ "hdEdxTotMIP_TPC" ], "dataSource": { "type": "direct", "query" : "inputTracks:TPC/TRACKS/0" @@ -73,7 +74,8 @@ "className": "o2::quality_control_modules::tpc::Tracks", "moduleName": "QcTPC", "detectorName": "TPC", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "300", + "movingWindows" : [ "h2DNClustersEta" ], "dataSource": { "type": "dataSamplingPolicy", "name": "tpc-tracks" From eab175696e7dde819c8ef783e7c4001debbcc552 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Wed, 13 Dec 2023 10:08:56 +0100 Subject: [PATCH 0287/1239] [WF CI] Test also QC and AnalysisQC --- MC/bin/tests/wf_test_pp.sh | 7 ++++--- test/run_workflow_tests.sh | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/MC/bin/tests/wf_test_pp.sh b/MC/bin/tests/wf_test_pp.sh index c5213a20f..4ce7068c7 100644 --- a/MC/bin/tests/wf_test_pp.sh +++ b/MC/bin/tests/wf_test_pp.sh @@ -15,9 +15,9 @@ # ----------- START ACTUAL JOB ----------------------------- -NSIGEVENTS=${NSIGEVENTS:-1} +NSIGEVENTS=${NSIGEVENTS:-5} SIGPROC=${SIGPROC:-cdiff} -NTIMEFRAMES=${NTIMEFRAMES:-1} +NTIMEFRAMES=${NTIMEFRAMES:-2} SIMENGINE=${SIMENGINE:-TGeant3} NWORKERS=${NWORKERS:-1} SEED=${SEED:-624} @@ -26,4 +26,5 @@ INTERACTIONRATE=${INTERACTIONRATE:-50000} # create workflow ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 5020 -col pp -gen pythia8 -proc ${SIGPROC} -ns ${NSIGEVENTS} \ -tf ${NTIMEFRAMES} -e ${SIMENGINE} -j ${NWORKERS} -seed ${SEED} \ - --include-analysis -run 310000 -interactionRate ${INTERACTIONRATE} + --include-analysis -run 310000 -interactionRate ${INTERACTIONRATE} \ + --include-local-qc diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index cbf05c418..2fc9bb487 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -67,11 +67,16 @@ test_single_wf() echo "Test ${wf_line} from ${wfs}" > ${LOG_FILE_WF} bash ${wf_script_local} >> ${LOG_FILE_WF} 2>&1 local ret_this=${?} + local ret_this_qc=0 + local ret_this_analysis=0 if [[ "${ret_this}" != "0" ]] ; then echo "[FATAL]: O2DPG_TEST Workflow creation failed" >> ${LOG_FILE_WF} elif [[ "${execute}" != "" ]] ; then ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 -tt aod >> ${LOG_FILE_WF} 2>&1 ret_this=${?} + [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } + [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; } + ret_this=$((ret_thi + ret_this_qc + ret_this_analysis)) [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} fi return ${ret_this} From 61e60a4ccd9ab0748d7b7be87ad947b9a1862338 Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Wed, 13 Dec 2023 10:19:27 +0100 Subject: [PATCH 0288/1239] Catch files that are WF related --- test/run_workflow_tests.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index 2fc9bb487..c4a2daeaa 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -76,7 +76,7 @@ test_single_wf() ret_this=${?} [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; } - ret_this=$((ret_thi + ret_this_qc + ret_this_analysis)) + ret_this=$((ret_this + ret_this_qc + ret_this_analysis)) [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} fi return ${ret_this} @@ -188,7 +188,9 @@ source ${REPO_DIR}/test/common/utils/utils.sh pushd ${REPO_DIR} > /dev/null # flag if anything changed in the sim workflow bin dir -changed_wf_bin=$(get_changed_files | grep MC/bin) +changed_wf_bin=$(get_changed_files | grep "MC/bin") +changed_wf_bin_related=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json") + # collect what has changed for PWGs collect_changed_pwg_wf_files @@ -252,7 +254,7 @@ pushd ${TEST_PARENT_DIR_BIN} > /dev/null # global return code for PWGs ret_global_bin=0 -if [[ "${changed_wf_bin}" != "" ]] ; then +if [[ "${changed_wf_bin}" != "" || "${changed_wf_bin_related}" != "" ]] ; then echo "### Test bin-related workflow creation ###" echo # Run all the bin test WF creations From 1cebe918be70f5a99a8e3479da8b1fc7bd6a97e3 Mon Sep 17 00:00:00 2001 From: David Date: Thu, 14 Dec 2023 23:55:14 -0500 Subject: [PATCH 0289/1239] Adding moving windows feature for some MFT A-QC objects --- DATA/production/qc-async/mft.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/mft.json b/DATA/production/qc-async/mft.json index 7cbb78b8c..d055da0d8 100644 --- a/DATA/production/qc-async/mft.json +++ b/DATA/production/qc-async/mft.json @@ -29,12 +29,13 @@ "className": "o2::quality_control_modules::mft::QcMFTClusterTask", "moduleName": "QcMFT", "detectorName": "MFT", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "300", "maxNumberCycles": "-1", "dataSource": { "type": "direct", "query": "randomcluster:MFT/COMPCLUSTERS/0;clustersrof:MFT/CLUSTERSROF/0;patterns:MFT/PATTERNS/0;cldict:MFT/CLUSDICT/0?lifetime=condition&ccdb-path=MFT/Calib/ClusterDictionary" }, + "movingWindows": [ "mClustersROFSize" ], "taskParameters": { "maxClusterROFSize" : "50000", "maxDuration" : "60000", @@ -58,7 +59,7 @@ "className": "o2::quality_control_modules::mft::QcMFTTrackTask", "moduleName": "QcMFT", "detectorName": "MFT", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "300", "maxNumberCycles": "-1", "dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.", "dataSource": { @@ -66,6 +67,7 @@ "query_comment" : "100% sampling", "query": "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0;clustersrofs:MFT/CLUSTERSROF/0" }, + "movingWindows": [ "mMFTTrackROFSize", "mMFTTrackEta", "mMFTTrackPhi" ], "taskParameters": { "ROFLengthInBC": "594", "MaxTrackROFSize": "10000", From 3dffb6b7d502ff9b19e620b2152b0053240d3587 Mon Sep 17 00:00:00 2001 From: afurs Date: Thu, 14 Dec 2023 20:35:19 +0300 Subject: [PATCH 0290/1239] FT0: restoring previous reco config for PbPb --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 98d915153..0b37841f5 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -460,7 +460,7 @@ export CONFIG_EXTRA_PROCESS_o2_trd_global_tracking+=";$ITSEXTRAERR;$TRACKTUNETPC # ad-hoc settings for FT0 export ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow="$ARGS_EXTRA_PROCESS_o2_ft0_reco_workflow --ft0-reconstructor" if [[ $BEAMTYPE == "PbPb" ]]; then - export CONFIG_EXTRA_PROCESS_o2_ft0_reco_workflow=";FT0TimeFilterParam.mAmpLower=20;" + export CONFIG_EXTRA_PROCESS_o2_ft0_reco_workflow=";FT0TimeFilterParam.mAmpLower=10;" fi # ad-hoc settings for FV0 From 192dac694a949ea59d07b2c694968cde7db9cc09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Fri, 15 Dec 2023 17:18:53 +0100 Subject: [PATCH 0291/1239] Extend autosetconverters (#1386) --- .../o2dpg_analysis_test_workflow.py | 33 +++++++------------ 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 703c8d446..75058219e 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -186,10 +186,12 @@ def get_additional_workflows(input_aod): from ROOT import TGrid TGrid.Connect("alien") froot = TFile.Open(input_aod, "READ") - found_O2collision_001 = False - found_O2zdc_001 = False - found_O2bc_001 = False - found_O2trackextra_001 = False + # Link of tables and converters + o2_analysis_converters = {"O2collision_001": "o2-analysis-collision-converter --doNotSwap", + "O2zdc_001": "o2-analysis-zdc-converter", + "O2bc_001": "o2-analysis-bc-converter", + "O2v0_001": "o2-analysis-v0converter", + "O2trackextra_001": "o2-analysis-tracks-extra-converter"} for i in froot.GetListOfKeys(): if "DF_" not in i.GetName(): continue @@ -197,23 +199,10 @@ def get_additional_workflows(input_aod): # print(i) for j in df_dir.GetListOfKeys(): # print(j) - if "O2collision_001" in j.GetName(): - found_O2collision_001 = True - if "O2zdc_001" in j.GetName(): - found_O2zdc_001 = True - if "O2bc_001" in j.GetName(): - found_O2bc_001 = True - if "O2trackextra_001" in j.GetName(): - found_O2trackextra_001 = True - if not found_O2collision_001: - additional_workflows.append("o2-analysis-collision-converter --doNotSwap") - if not found_O2zdc_001: - additional_workflows.append("o2-analysis-zdc-converter") - if not found_O2bc_001: - additional_workflows.append("o2-analysis-bc-converter") - if not found_O2trackextra_001: - additional_workflows.append("o2-analysis-tracks-extra-converter") - break + if j.GetName() in o2_analysis_converters: + o2_analysis_converters.pop(j.GetName()) + for i in o2_analysis_converters: + additional_workflows.append(o2_analysis_converters[i]) return additional_workflows @@ -242,7 +231,7 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis input_aod = f"@{input_aod}" additional_workflows = [] - if autoset_converters: # This is needed to run with the latest TAG of the O2Physics with the older data + if autoset_converters: # This is needed to run with the latest TAG of the O2Physics with the older data additional_workflows = get_additional_workflows(input_aod) data_or_mc = ANALYSIS_VALID_MC if is_mc else ANALYSIS_VALID_DATA From 974703bf2666ed64c48aaf3b99eda4a992de5e7e Mon Sep 17 00:00:00 2001 From: Francesco Mazzaschi <43742195+fmazzasc@users.noreply.github.com> Date: Sun, 17 Dec 2023 11:23:41 +0100 Subject: [PATCH 0292/1239] Add TPC as default source for the svertexer (#1391) --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 70d81c64f..626d81b48 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1320,7 +1320,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # Take None as default, we only add more if nothing from anchorConfig svfinder_sources = anchorConfig.get("o2-secondary-vertexing-workflow-options",{}).get("vertexing-sources", None) if not svfinder_sources: - svfinder_sources = "ITS,ITS-TPC,TPC-TRD,TPC-TOF,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" + svfinder_sources = "ITS,TPC,ITS-TPC,TPC-TRD,TPC-TOF,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" if isActive("MID"): svfinder_sources += ",MID" SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] From 10fabf91bdce98a29be10fbf92f2ce0794825f77 Mon Sep 17 00:00:00 2001 From: Andreas Molander Date: Mon, 18 Dec 2023 10:22:45 +0200 Subject: [PATCH 0293/1239] FIT: add digitizer settings for MC anchored to 2023 PbPb Provide FT0 and FV0 gain settings for MC anchored to 2023 PbPb. This is temporary and will be removed when these settings are pulled from CCDB. --- MC/bin/o2dpg_sim_config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index 01692d8ff..d065998e8 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -85,6 +85,11 @@ def add(cfg, flatconfig): if args.col == "PbPb" or (args.embedding and args.colBkg == "PbPb"): add(config, {"ITSVertexerParam.lowMultBeamDistCut": "0."}) + # FIT digitizer settings for 2023 PbPb + if 543437 <= int(args.run) and int(args.run) <= 545367: + add(config, {"FT0DigParam.mMip_in_V": "7", "FT0DigParam.mMV_2_Nchannels": "2", "FT0DigParam.mMV_2_NchannelsInverse": "0.5"}) + add(config, {"FV0DigParam.adcChannelsPerMip": "4"}) + return config From 8df18779bca9fbd5b6f6374d9652dfa33a3e4d06 Mon Sep 17 00:00:00 2001 From: Jian Liu Date: Mon, 18 Dec 2023 17:09:45 +0100 Subject: [PATCH 0294/1239] Add script for updating qc configuration and replace all MC qc jsons to alice-ccdb (#1369) * add script for updating qc configuration and replaced all MC qc jsons * fix a typo * embed the template into the script * overrite ccdb url * reupdate jsons --------- Co-authored-by: JianLIUhep --- MC/bin/o2dpg_qc_finalization_workflow.py | 11 +- MC/bin/o2dpg_sim_workflow.py | 5 +- .../json/ITSTPCmatchedTracks_direct_MC.json | 88 +++--- MC/config/QC/json/cpv-clusters-task.json | 4 +- MC/config/QC/json/cpv-digits-task.json | 4 +- MC/config/QC/json/emc-bc-task.json | 90 +++--- MC/config/QC/json/emc-reco-tasks.json | 6 +- MC/config/QC/json/event-selection-qa.json | 78 ++--- MC/config/QC/json/event-track-qa.json | 282 +++++++++--------- .../QC/json/ft0-reconstruction-config.json | 77 +++-- MC/config/QC/json/its-clusters-tracks-qc.json | 46 +-- MC/config/QC/json/its-mc-tracks-qc.json | 71 +++-- MC/config/QC/json/mft-clusters.json | 47 +-- MC/config/QC/json/mft-digits-0.json | 39 +-- MC/config/QC/json/mft-digits-1.json | 39 +-- MC/config/QC/json/mft-digits-2.json | 39 +-- MC/config/QC/json/mft-digits-3.json | 39 +-- MC/config/QC/json/mft-digits-4.json | 39 +-- MC/config/QC/json/mft-tracks.json | 9 +- .../QC/json/phs-cells-clusters-task.json | 7 +- MC/config/QC/json/pidft0tof.json | 135 ++++----- MC/config/QC/json/pidft0tofNoTRD.json | 135 ++++----- MC/config/QC/json/pidtof.json | 133 ++++----- MC/config/QC/json/pidtofNoTRD.json | 133 ++++----- MC/config/QC/json/tof-trending-hits.json | 12 +- .../tofMatchedTracks_AllTypes_direct_MC.json | 103 +++---- ...chedTracks_ITSTPCTOF_TPCTOF_direct_MC.json | 105 ++++--- MC/config/QC/json/tofdigits.json | 12 +- MC/config/QC/json/tpc-qc-standard-direct.json | 48 +-- MC/config/QC/json/tpc-qc-tracking-direct.json | 14 +- MC/config/QC/json/trd-digits-task.json | 13 +- MC/config/QC/json/vertexing-qc-direct-mc.json | 109 ++++--- UTILS/update-qc-json.py | 62 ++++ 33 files changed, 1041 insertions(+), 993 deletions(-) create mode 100644 UTILS/update-qc-json.py diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 415c789ca..1b2866944 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -33,7 +33,7 @@ def QC_finalize_name(name): return name + "_finalize" qcdir = "QC" -def include_all_QC_finalization(ntimeframes, standalone, run, productionTag): +def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB): stages = [] @@ -49,7 +49,7 @@ def add_QC_finalization(taskName, qcConfigPath, needs=None): task = createTask(name=QC_finalize_name(taskName), needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') task['cmd'] = f'o2-qc --config {qcConfigPath} --remote-batch {taskName}.root' + \ - f' --override-values "qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag}"' + \ + f' --override-values "qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag};qc.config.conditionDB.url={conditionDB}"' + \ ' ' + getDPL_global_options() stages.append(task) @@ -66,7 +66,8 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif task = createTask(name=taskName, needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') overrideValues = '--override-values "' overrideValues += f'qc.config.Activity.number={run};' if runSpecific else 'qc.config.Activity.number=0;' - overrideValues += f'qc.config.Activity.periodName={productionTag}"' if prodSpecific else 'qc.config.Activity.periodName="' + overrideValues += f'qc.config.Activity.periodName={productionTag};' if prodSpecific else 'qc.config.Activity.periodName=;' + overrideValues += f'qc.config.conditionDB.url={conditionDB}"' task['cmd'] = f'o2-qc --config {qcConfigPath} ' + \ overrideValues + ' ' + getDPL_global_options() stages.append(task) @@ -117,7 +118,7 @@ def main() -> int: parser.add_argument('-o',help='output workflow file', default='workflow.json') parser.add_argument('-run',help="Run number for this MC", default=300000) parser.add_argument('-productionTag',help="Production tag for this MC", default='unknown') - + parser.add_argument('-conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') args = parser.parse_args() print (args) @@ -139,7 +140,7 @@ def main() -> int: mkdir(qcdir) workflow={} - workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag) + workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB) dump_workflow(workflow["stages"], args.o) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 626d81b48..72b0adb8f 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -48,6 +48,7 @@ # - some external tool should sample it within # - we can also sample it ourselfs here parser.add_argument('--timestamp', type=int, help="Anchoring timestamp (defaults to now)", default=-1) +parser.add_argument('--conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') parser.add_argument('--condition-not-after', type=int, help="only consider CCDB objects not created after this timestamp (for TimeMachine)", default=3385078236000) parser.add_argument('--orbitsPerTF', type=int, help="Timeframe size in number of LHC orbits", default=128) parser.add_argument('--anchor-config',help="JSON file to contextualise workflow with external configs (config values etc.) for instance comping from data reco workflows.", default='') @@ -1173,7 +1174,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # the --local-batch argument will make QC Tasks store their results in a file and merge with any existing objects task['cmd'] = f'{readerCommand} | o2-qc --config {configFilePath}' + \ f' --local-batch ../{qcdir}/{objectsFile}' + \ - f' --override-values "qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp}"' + \ + f' --override-values "qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \ ' ' + getDPL_global_options(ccdbbackend=False) # Prevents this task from being run for multiple TimeFrames at the same time, thus trying to modify the same file. task['semaphore'] = objectsFile @@ -1445,7 +1446,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): job_merging = False if includeFullQC: - workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag)) + workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB)) if includeAnalysis: diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index d29c247c0..95d25f655 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -1,51 +1,46 @@ { - "qc" : { - "config" : { - "database" : { - "implementation" : "CCDB", - "host" : "ccdb-test.cern.ch:8080", - "username" : "not_applicable", - "password" : "not_applicable", - "name" : "not_applicable" + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "Activity" : { - "number" : "42", - "type" : "2", + "Activity": { + "number": "42", + "type": "2", "provenance": "qc_mc", "passName": "passMC", "periodName": "SimChallenge" }, - "monitoring" : { - "url" : "no-op://" + "monitoring": { + "url": "no-op://" }, - "consul" : { - "url" : "" + "consul": { + "url": "" }, - "conditionDB" : { - "url" : "ccdb-test.cern.ch:8080" - }, - "infologger" : { "" : "Configuration of the Infologger (optional).", - "filterDiscardDebug" : "false", - "" : "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel" : "21", - "" : "Message at this level or above are discarded (default: 21 - Trace)" } + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "tasks" : { - "MatchTrITSTPC" : { - "active" : "true", - "className" : "o2::quality_control_modules::glo::ITSTPCMatchingTask", - "moduleName" : "QcGLO", - "detectorName" : "GLO", - "cycleDurationSeconds" : "3600", - "maxNumberCycles" : "-1", - "dataSource" : { - "type" : "direct", - "query_comment" : "checking every matched track", - "query" : "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackITS:ITS/TRACKS/0;trackITSROF:ITS/ITSTrackROF/0;trackITSClIdx:ITS/TRACKCLSID/0;alpparITS:ITS/ALPIDEPARAM/0?lifetime=condition&ccdb-path=ITS/Config/AlpideParam;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;trackITSMCTR:ITS/TRACKSMCTR" + "tasks": { + "MatchTrITSTPC": { + "active": "true", + "className": "o2::quality_control_modules::glo::ITSTPCMatchingTask", + "moduleName": "QcGLO", + "detectorName": "GLO", + "cycleDurationSeconds": "3600", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS;trackTPCClRefs:TPC/CLUSREFS;trackITS:ITS/TRACKS/0;trackITSROF:ITS/ITSTrackROF/0;trackITSClIdx:ITS/TRACKCLSID/0;alpparITS:ITS/ALPIDEPARAM/0?lifetime=condition&ccdb-path=ITS/Config/AlpideParam;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;trackITSMCTR:ITS/TRACKSMCTR" }, - "taskParameters" : { - "GID" : "ITS-TPC,ITS", - "verbose" : "false", + "taskParameters": { + "GID": "ITS-TPC,ITS", + "verbose": "false", "minPtITSCut": "0.f", "etaITSCut": "1e10f", "minNITSClustersCut": "0", @@ -58,9 +53,9 @@ "minPtCut": "0.f", "maxPtCut": "1e10f", "etaCut": "1e10f", - "isMC" : "true" + "isMC": "true" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -68,14 +63,13 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" }, - "location" : "remote", - "saveObjectsToFile" : "ITSTPCmatched.root", - "" : "For debugging, path to the file where to save. If empty or missing it won't save." + "location": "remote", + "saveObjectsToFile": "ITSTPCmatched.root", + "": "For debugging, path to the file where to save. If empty or missing it won't save." } } }, - "dataSamplingPolicies" : [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/cpv-clusters-task.json b/MC/config/QC/json/cpv-clusters-task.json index 6292682ac..5d03bc94e 100644 --- a/MC/config/QC/json/cpv-clusters-task.json +++ b/MC/config/QC/json/cpv-clusters-task.json @@ -16,13 +16,13 @@ "periodName": "SimChallenge" }, "monitoring": { - "url": "infologger:///debug?qc" + "url": "no-op://" }, "consul": { "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { diff --git a/MC/config/QC/json/cpv-digits-task.json b/MC/config/QC/json/cpv-digits-task.json index bf6d788ef..2fd5f3c30 100644 --- a/MC/config/QC/json/cpv-digits-task.json +++ b/MC/config/QC/json/cpv-digits-task.json @@ -16,13 +16,13 @@ "periodName": "SimChallenge" }, "monitoring": { - "url": "infologger:///debug?qc" + "url": "no-op://" }, "consul": { "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { diff --git a/MC/config/QC/json/emc-bc-task.json b/MC/config/QC/json/emc-bc-task.json index dd07cfa09..7cb788561 100644 --- a/MC/config/QC/json/emc-bc-task.json +++ b/MC/config/QC/json/emc-bc-task.json @@ -1,49 +1,49 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "alice-ccdb.cern.ch" - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "tasks": { - "BCTaskEMCAL": { - "active": "true", - "taskName": "BCs", - "className": "o2::quality_control_modules::emcal::BCTask", - "moduleName": "QcEMCAL", - "detectorName": "EMC", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" - }, - "taskParameters": { - "AliasMB":"MINBIAS_TVX_L0,C0TVX,CMTVXTSC,C0TVXTSC,C0TVXTCE" - } - } + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" }, - "checks": {} + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "BCTaskEMCAL": { + "active": "true", + "taskName": "BCs", + "className": "o2::quality_control_modules::emcal::BCTask", + "moduleName": "QcEMCAL", + "detectorName": "EMC", + "cycleDurationSeconds": "60", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "emcal-triggers:EMC/CELLSTRGR/0;ctp-digits:CTP/DIGITS;ctp-config:CTP/CONFIG/0?lifetime=condition&ccdb-path=CTP/Config/Config&ccdb-run-dependent=true" + }, + "taskParameters": { + "AliasMB": "MINBIAS_TVX_L0,C0TVX,CMTVXTSC,C0TVXTSC,C0TVXTCE" + } + } }, - "dataSamplingPolicies": [] - } \ No newline at end of file + "checks": {} + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/emc-reco-tasks.json b/MC/config/QC/json/emc-reco-tasks.json index 550943f55..1c99fb4b1 100644 --- a/MC/config/QC/json/emc-reco-tasks.json +++ b/MC/config/QC/json/emc-reco-tasks.json @@ -64,7 +64,7 @@ "hasInvMassMesons": "true", "mesonClustersRejectExotics": 1 }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Ideal", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -72,11 +72,11 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" } } }, "checks": {} }, "dataSamplingPolicies": [] -} +} \ No newline at end of file diff --git a/MC/config/QC/json/event-selection-qa.json b/MC/config/QC/json/event-selection-qa.json index 5a662309a..aafa2f51a 100644 --- a/MC/config/QC/json/event-selection-qa.json +++ b/MC/config/QC/json/event-selection-qa.json @@ -1,40 +1,40 @@ { - "internal-dpl-aod-index-builder": "", - "internal-dpl-aod-reader": { - "time-limit": "0", - "orbit-offset-enumeration": "0", - "orbit-multiplier-enumeration": "0", - "start-value-enumeration": "0", - "end-value-enumeration": "-1", - "step-value-enumeration": "1", - "aod-file": "../AO2D.root" - }, - "internal-dpl-aod-spawner": "", - "timestamp-task": { - "verbose": "false", - "rct-path": "RCT\/RunInformation\/", - "start-orbit-path": "GRP\/StartOrbit", - "ccdb-url": "http:\/\/alice-ccdb.cern.ch", - "isRun2MC": "false" - }, - "bc-selection-task": { - "processRun2": "false", - "processRun3": "true" - }, - "event-selection-task": { - "syst": "pp", - "muonSelection": "0", - "customDeltaBC": "300", - "isMC": "true", - "processRun2": "false", - "processRun3": "true" - }, - "event-selection-qa-task": { - "isMC": "true", - "processRun2": "false", - "processRun3": "true", - "processMCRun3": "true" - }, - "internal-dpl-injected-dummy-sink": "", - "internal-dpl-aod-global-analysis-file-sink": "" -} + "internal-dpl-aod-index-builder": "", + "internal-dpl-aod-reader": { + "time-limit": "0", + "orbit-offset-enumeration": "0", + "orbit-multiplier-enumeration": "0", + "start-value-enumeration": "0", + "end-value-enumeration": "-1", + "step-value-enumeration": "1", + "aod-file": "../AO2D.root" + }, + "internal-dpl-aod-spawner": "", + "timestamp-task": { + "verbose": "false", + "rct-path": "RCT/RunInformation/", + "start-orbit-path": "GRP/StartOrbit", + "ccdb-url": "http://alice-ccdb.cern.ch", + "isRun2MC": "false" + }, + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-task": { + "syst": "pp", + "muonSelection": "0", + "customDeltaBC": "300", + "isMC": "true", + "processRun2": "false", + "processRun3": "true" + }, + "event-selection-qa-task": { + "isMC": "true", + "processRun2": "false", + "processRun3": "true", + "processMCRun3": "true" + }, + "internal-dpl-injected-dummy-sink": "", + "internal-dpl-aod-global-analysis-file-sink": "" +} \ No newline at end of file diff --git a/MC/config/QC/json/event-track-qa.json b/MC/config/QC/json/event-track-qa.json index 2a28f345a..303772188 100644 --- a/MC/config/QC/json/event-track-qa.json +++ b/MC/config/QC/json/event-track-qa.json @@ -1,142 +1,142 @@ { - "bc-selection-task": { - "processRun2": "false", - "processRun3": "true" - }, - "qa-tracking-efficiency-data": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true" - }, - "qa-tracking-efficiency-muon": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-pion": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-electron": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-kaon": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-proton": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-deuteron": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-triton": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "qa-tracking-efficiency-helium3": { - "eta-min": "-0.8", - "eta-max": "0.8", - "pt-min": "0.1", - "pt-max": "10", - "track-sel": "true", - "pt-bins": "500", - "log-pt": "1", - "eta-bins": "500", - "phi-bins": "500", - "make-eff": "true", - "sel-prim": "true" - }, - "event-selection-task": { - "syst": "pp", - "muonSelection": "0", - "isMC": "true", - "processRun2": "false", - "processRun3": "true" - }, - "track-extension": { - "processRun2": "false", - "processRun3": "true" - }, - "track-selection": { - "isRun3": "true" - }, - "qa-event-track": { - "isMC": "true", - "isRun3": "true", - "processData": "false", - "processMC": "true" - } -} + "bc-selection-task": { + "processRun2": "false", + "processRun3": "true" + }, + "qa-tracking-efficiency-data": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true" + }, + "qa-tracking-efficiency-muon": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-pion": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-electron": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-kaon": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-proton": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-deuteron": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-triton": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "qa-tracking-efficiency-helium3": { + "eta-min": "-0.8", + "eta-max": "0.8", + "pt-min": "0.1", + "pt-max": "10", + "track-sel": "true", + "pt-bins": "500", + "log-pt": "1", + "eta-bins": "500", + "phi-bins": "500", + "make-eff": "true", + "sel-prim": "true" + }, + "event-selection-task": { + "syst": "pp", + "muonSelection": "0", + "isMC": "true", + "processRun2": "false", + "processRun3": "true" + }, + "track-extension": { + "processRun2": "false", + "processRun3": "true" + }, + "track-selection": { + "isRun3": "true" + }, + "qa-event-track": { + "isMC": "true", + "isRun3": "true", + "processData": "false", + "processMC": "true" + } +} \ No newline at end of file diff --git a/MC/config/QC/json/ft0-reconstruction-config.json b/MC/config/QC/json/ft0-reconstruction-config.json index 0f8f770cf..bebf3faf5 100644 --- a/MC/config/QC/json/ft0-reconstruction-config.json +++ b/MC/config/QC/json/ft0-reconstruction-config.json @@ -1,44 +1,43 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" }, - "tasks": { - "RecPoints": { - "active": "true", - "className": "o2::quality_control_modules::ft0::RecPointsQcTask", - "moduleName": "QcFT0", - "detectorName": "FT0", - "cycleDurationSeconds": "600", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query": "recpoints:FT0/RECPOINTS/0;channels:FT0/RECCHDATA/0" - } + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "RecPoints": { + "active": "true", + "className": "o2::quality_control_modules::ft0::RecPointsQcTask", + "moduleName": "QcFT0", + "detectorName": "FT0", + "cycleDurationSeconds": "600", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query": "recpoints:FT0/RECPOINTS/0;channels:FT0/RECCHDATA/0" } - } } -} + } +} \ No newline at end of file diff --git a/MC/config/QC/json/its-clusters-tracks-qc.json b/MC/config/QC/json/its-clusters-tracks-qc.json index 5c0cef231..5fa4ec5ff 100644 --- a/MC/config/QC/json/its-clusters-tracks-qc.json +++ b/MC/config/QC/json/its-clusters-tracks-qc.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -41,12 +41,12 @@ "taskParameters": { "layer": "1111111", "nThreads": "1", - "nBCbins" : "103", - "dicttimestamp" : "0", - "publishSummary1D": "0", - "publishDetailedSummary": "1" + "nBCbins": "103", + "dicttimestamp": "0", + "publishSummary1D": "0", + "publishDetailedSummary": "1" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -54,7 +54,7 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" } }, "Tracks": { @@ -74,11 +74,11 @@ "vertexXYsize": "0.5", "vertexZsize": "15", "vertexRsize": "0.8", - "NtracksMAX" : "5000", + "NtracksMAX": "5000", "doTTree": "0", - "nBCbins" : "103", - "dicttimestamp" : "0", - "doNorm": "1" + "nBCbins": "103", + "dicttimestamp": "0", + "doNorm": "1" } } }, @@ -90,15 +90,15 @@ "policy": "OnEachSeparately", "detectorName": "ITS", "checkParameters": { - "maxcluoccL0" : "5", - "maxcluoccL1" : "4", - "maxcluoccL2" : "3", - "maxcluoccL3" : "2", - "maxcluoccL4" : "1", - "maxcluoccL5" : "1", - "maxcluoccL6" : "1", - "skipxbinsoccupancy" : "", - "skipybinsoccupancy" : "" + "maxcluoccL0": "5", + "maxcluoccL1": "4", + "maxcluoccL2": "3", + "maxcluoccL3": "2", + "maxcluoccL4": "1", + "maxcluoccL5": "1", + "maxcluoccL6": "1", + "skipxbinsoccupancy": "", + "skipybinsoccupancy": "" }, "dataSource": [ { @@ -141,11 +141,11 @@ "VertexCoordinates", "VertexRvsZ", "VertexZ", - "BunchCrossingIDvsClusterRatio" + "BunchCrossingIDvsClusterRatio" ] } ] } } - } -} + } +} \ No newline at end of file diff --git a/MC/config/QC/json/its-mc-tracks-qc.json b/MC/config/QC/json/its-mc-tracks-qc.json index 187c2110d..97cb8846c 100644 --- a/MC/config/QC/json/its-mc-tracks-qc.json +++ b/MC/config/QC/json/its-mc-tracks-qc.json @@ -1,49 +1,48 @@ { - "qc" : { - "config" : { - "database" : { - "implementation" : "CCDB", - "host" : "ccdb-test.cern.ch:8080", - "username" : "not_applicable", - "password" : "not_applicable", - "name" : "not_applicable" + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "Activity" : { - "number" : "42", - "type" : "2", + "Activity": { + "number": "42", + "type": "2", "provenance": "qc_mc", "passName": "passMC", "periodName": "SimChallenge" }, - "monitoring" : { - "url" : "no-op://" + "monitoring": { + "url": "no-op://" }, - "consul" : { - "url" : "" + "consul": { + "url": "" }, - "conditionDB" : { - "url" : "ccdb-test.cern.ch:8080" + "conditionDB": { + "url": "alice-ccdb.cern.ch" } }, - "tasks" : { - "TracksMc" : { - "active" : "true", - "className" : "o2::quality_control_modules::its::ITSTrackSimTask", - "moduleName" : "QcITS", - "detectorName" : "ITS", - "cycleDurationSeconds" : "30", - "maxNumberCycles" : "-1", - "dataSource_comment" : "The other type of dataSource is \"direct\", see basic-no-sampling.json.", - "dataSource" : { - "type" : "direct", - "query" : "tracks:ITS/TRACKS/0;mstruth:ITS/TRACKSMCTR/0;compclus:ITS/COMPCLUSTERS/0;mcclustruth:ITS/CLUSTERSMCTR/0" + "tasks": { + "TracksMc": { + "active": "true", + "className": "o2::quality_control_modules::its::ITSTrackSimTask", + "moduleName": "QcITS", + "detectorName": "ITS", + "cycleDurationSeconds": "30", + "maxNumberCycles": "-1", + "dataSource_comment": "The other type of dataSource is \"direct\", see basic-no-sampling.json.", + "dataSource": { + "type": "direct", + "query": "tracks:ITS/TRACKS/0;mstruth:ITS/TRACKSMCTR/0;compclus:ITS/COMPCLUSTERS/0;mcclustruth:ITS/CLUSTERSMCTR/0" }, - "taskParameters" : { - "O2GrpPath" : "./o2sim_grp.root", - "collisionsContextPath": "./collisioncontext.root" - } - + "taskParameters": { + "O2GrpPath": "./o2sim_grp.root", + "collisionsContextPath": "./collisioncontext.root" + } } } - } -} + } +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-clusters.json b/MC/config/QC/json/mft-clusters.json index 896ad635a..ba03a1e9d 100644 --- a/MC/config/QC/json/mft-clusters.json +++ b/MC/config/QC/json/mft-clusters.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -38,13 +38,13 @@ "type": "direct", "query": "randomcluster:MFT/COMPCLUSTERS/0;clustersrof:MFT/CLUSTERSROF/0;patterns:MFT/PATTERNS/0;cldict:MFT/CLUSDICT/0?lifetime=condition&ccdb-path=MFT/Calib/ClusterDictionary" }, - "taskParameters" : { - "maxClusterROFSize" : "50000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "taskParameters": { + "maxClusterROFSize": "50000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -52,22 +52,30 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" - }, + "needPropagatorD": "false" + }, "location": "remote" } }, "checks": { "Clusters": { "active": "true", - "dataSource": [{ - "type": "Task", - "name": "Clusters", - "MOs" : ["mClusterOccupancy","mClusterPatternIndex","mClusterSizeSummary","mGroupedClusterSizeSummary","mClusterOccupancySummary"] - }], - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "dataSource": [ + { + "type": "Task", + "name": "Clusters", + "MOs": [ + "mClusterOccupancy", + "mClusterPatternIndex", + "mClusterSizeSummary", + "mGroupedClusterSizeSummary", + "mClusterOccupancySummary" + ] + } + ], + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, "className": "o2::quality_control_modules::mft::QcMFTClusterCheck", "moduleName": "QcMFT", @@ -76,6 +84,5 @@ } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-digits-0.json b/MC/config/QC/json/mft-digits-0.json index ce6ab23dc..c23beec15 100644 --- a/MC/config/QC/json/mft-digits-0.json +++ b/MC/config/QC/json/mft-digits-0.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -40,11 +40,11 @@ }, "taskParameters": { "FLP": "0", - "NoiseScan" : "0", - "maxDigitROFSize" : "5000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "NoiseScan": "0", + "maxDigitROFSize": "5000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, "location": "remote" } @@ -56,18 +56,23 @@ "moduleName": "QcMFT", "detectorName": "MFT", "policy": "OnEachSeparately", - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, - "dataSource": [{ - "type": "Task", - "name": "Digits", - "MOs" : ["mDigitChipOccupancy","mDigitOccupancySummary","mDigitChipStdDev"] - }] + "dataSource": [ + { + "type": "Task", + "name": "Digits", + "MOs": [ + "mDigitChipOccupancy", + "mDigitOccupancySummary", + "mDigitChipStdDev" + ] + } + ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-digits-1.json b/MC/config/QC/json/mft-digits-1.json index d4219645c..59d3ea277 100644 --- a/MC/config/QC/json/mft-digits-1.json +++ b/MC/config/QC/json/mft-digits-1.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -40,11 +40,11 @@ }, "taskParameters": { "FLP": "1", - "NoiseScan" : "0", - "maxDigitROFSize" : "5000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "NoiseScan": "0", + "maxDigitROFSize": "5000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, "location": "remote" } @@ -56,18 +56,23 @@ "moduleName": "QcMFT", "detectorName": "MFT", "policy": "OnEachSeparately", - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, - "dataSource": [{ - "type": "Task", - "name": "Digits", - "MOs" : ["mDigitChipOccupancy","mDigitOccupancySummary","mDigitChipStdDev"] - }] + "dataSource": [ + { + "type": "Task", + "name": "Digits", + "MOs": [ + "mDigitChipOccupancy", + "mDigitOccupancySummary", + "mDigitChipStdDev" + ] + } + ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-digits-2.json b/MC/config/QC/json/mft-digits-2.json index e8f3e6c23..8ae2a2d58 100644 --- a/MC/config/QC/json/mft-digits-2.json +++ b/MC/config/QC/json/mft-digits-2.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -40,11 +40,11 @@ }, "taskParameters": { "FLP": "2", - "NoiseScan" : "0", - "maxDigitROFSize" : "5000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "NoiseScan": "0", + "maxDigitROFSize": "5000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, "location": "remote" } @@ -56,18 +56,23 @@ "moduleName": "QcMFT", "detectorName": "MFT", "policy": "OnEachSeparately", - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, - "dataSource": [{ - "type": "Task", - "name": "Digits", - "MOs" : ["mDigitChipOccupancy","mDigitOccupancySummary","mDigitChipStdDev"] - }] + "dataSource": [ + { + "type": "Task", + "name": "Digits", + "MOs": [ + "mDigitChipOccupancy", + "mDigitOccupancySummary", + "mDigitChipStdDev" + ] + } + ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-digits-3.json b/MC/config/QC/json/mft-digits-3.json index e581d42a1..287edbb97 100644 --- a/MC/config/QC/json/mft-digits-3.json +++ b/MC/config/QC/json/mft-digits-3.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -40,11 +40,11 @@ }, "taskParameters": { "FLP": "3", - "NoiseScan" : "0", - "maxDigitROFSize" : "5000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "NoiseScan": "0", + "maxDigitROFSize": "5000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, "location": "remote" } @@ -56,18 +56,23 @@ "moduleName": "QcMFT", "detectorName": "MFT", "policy": "OnEachSeparately", - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, - "dataSource": [{ - "type": "Task", - "name": "Digits", - "MOs" : ["mDigitChipOccupancy","mDigitOccupancySummary","mDigitChipStdDev"] - }] + "dataSource": [ + { + "type": "Task", + "name": "Digits", + "MOs": [ + "mDigitChipOccupancy", + "mDigitOccupancySummary", + "mDigitChipStdDev" + ] + } + ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-digits-4.json b/MC/config/QC/json/mft-digits-4.json index 62068af25..a544794ce 100644 --- a/MC/config/QC/json/mft-digits-4.json +++ b/MC/config/QC/json/mft-digits-4.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -40,11 +40,11 @@ }, "taskParameters": { "FLP": "4", - "NoiseScan" : "0", - "maxDigitROFSize" : "5000", - "maxDuration" : "60000", - "timeBinSize" : "0.1", - "ROFLengthInBC" : "594" + "NoiseScan": "0", + "maxDigitROFSize": "5000", + "maxDuration": "60000", + "timeBinSize": "0.1", + "ROFLengthInBC": "594" }, "location": "remote" } @@ -56,18 +56,23 @@ "moduleName": "QcMFT", "detectorName": "MFT", "policy": "OnEachSeparately", - "checkParameters" : { - "ZoneThresholdMedium" : "2", - "ZoneThresholdBad" : "5" + "checkParameters": { + "ZoneThresholdMedium": "2", + "ZoneThresholdBad": "5" }, - "dataSource": [{ - "type": "Task", - "name": "Digits", - "MOs" : ["mDigitChipOccupancy","mDigitOccupancySummary","mDigitChipStdDev"] - }] + "dataSource": [ + { + "type": "Task", + "name": "Digits", + "MOs": [ + "mDigitChipOccupancy", + "mDigitOccupancySummary", + "mDigitChipStdDev" + ] + } + ] } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/mft-tracks.json b/MC/config/QC/json/mft-tracks.json index 548dc8243..718c1e581 100644 --- a/MC/config/QC/json/mft-tracks.json +++ b/MC/config/QC/json/mft-tracks.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -37,7 +37,7 @@ "type": "direct", "query": "tracks:MFT/TRACKS/0;tracksrofs:MFT/MFTTrackROF/0;clustersrofs:MFT/CLUSTERSROF/0" }, - "taskParameters" : { + "taskParameters": { "ROFLengthInBC": "594", "MaxTrackROFSize": "10000", "MaxClusterROFSize": "50000", @@ -49,6 +49,5 @@ }, "checks": {} }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/phs-cells-clusters-task.json b/MC/config/QC/json/phs-cells-clusters-task.json index bb56a398e..acff6b067 100644 --- a/MC/config/QC/json/phs-cells-clusters-task.json +++ b/MC/config/QC/json/phs-cells-clusters-task.json @@ -9,21 +9,18 @@ "name": "not_applicable" }, "Activity": { - "number": "43", + "number": "42", "type": "2", "provenance": "qc_mc", "passName": "passMC", "periodName": "SimChallenge" }, "monitoring": { - "url": "infologger:///debug?qc" + "url": "no-op://" }, "consul": { "url": "" }, - "#conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, "conditionDB": { "url": "alice-ccdb.cern.ch" } diff --git a/MC/config/QC/json/pidft0tof.json b/MC/config/QC/json/pidft0tof.json index 8b292a7ef..13f115001 100644 --- a/MC/config/QC/json/pidft0tof.json +++ b/MC/config/QC/json/pidft0tof.json @@ -1,73 +1,66 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, - "infologger": { - "": "Configuration of the Infologger (optional).", - "filterDiscardDebug": "false", - "": "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel": "21", - "": "Message at this level or above are discarded (default: 21 - Trace)" - } - }, - "tasks": { - "PID": { - "active": "true", - "className": "o2::quality_control_modules::pid::TaskFT0TOF", - "moduleName": "QcTOF", - "detectorName": "TOF", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query_comment": "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0;recpoints:FT0/RECPOINTS/0" - }, - "taskParameters": { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", - "verbose": "false", - "minPtCut": "0.3f", - "etaCut": "0.8f", - "useFT0": "true", - "minNTPCClustersCut": "60", - "minDCACut": "100.f", - "minDCACutY": "10.f" - }, - "grpGeomRequest" : { - "geomRequest": "Aligned", - "askGRPECS": "false", - "askGRPLHCIF": "false", - "askGRPMagField": "true", - "askMatLUT": "false", - "askTime": "false", - "askOnceAllButField": "true", - "needPropagatorD": "false" - }, - "": "For debugging, path to the file where to save. If empty or missing it won't save." - } - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "dataSamplingPolicies": [] -} + "tasks": { + "PID": { + "active": "true", + "className": "o2::quality_control_modules::pid::TaskFT0TOF", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0;recpoints:FT0/RECPOINTS/0" + }, + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", + "verbose": "false", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "useFT0": "true", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + }, + "": "For debugging, path to the file where to save. If empty or missing it won't save." + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/pidft0tofNoTRD.json b/MC/config/QC/json/pidft0tofNoTRD.json index da568ad46..c72a2e5c5 100644 --- a/MC/config/QC/json/pidft0tofNoTRD.json +++ b/MC/config/QC/json/pidft0tofNoTRD.json @@ -1,73 +1,66 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, - "infologger": { - "": "Configuration of the Infologger (optional).", - "filterDiscardDebug": "false", - "": "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel": "21", - "": "Message at this level or above are discarded (default: 21 - Trace)" - } - }, - "tasks": { - "PID": { - "active": "true", - "className": "o2::quality_control_modules::pid::TaskFT0TOF", - "moduleName": "QcTOF", - "detectorName": "TOF", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query_comment": "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;recpoints:FT0/RECPOINTS/0" - }, - "taskParameters": { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", - "verbose": "false", - "minPtCut": "0.3f", - "etaCut": "0.8f", - "useFT0": "true", - "minNTPCClustersCut": "60", - "minDCACut": "100.f", - "minDCACutY": "10.f" - }, - "grpGeomRequest" : { - "geomRequest": "Aligned", - "askGRPECS": "false", - "askGRPLHCIF": "false", - "askGRPMagField": "true", - "askMatLUT": "false", - "askTime": "false", - "askOnceAllButField": "true", - "needPropagatorD": "false" - }, - "": "For debugging, path to the file where to save. If empty or missing it won't save." - } - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "dataSamplingPolicies": [] -} + "tasks": { + "PID": { + "active": "true", + "className": "o2::quality_control_modules::pid::TaskFT0TOF", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;recpoints:FT0/RECPOINTS/0" + }, + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", + "verbose": "false", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "useFT0": "true", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + }, + "": "For debugging, path to the file where to save. If empty or missing it won't save." + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/pidtof.json b/MC/config/QC/json/pidtof.json index 22b75c9d5..ac23d75cc 100644 --- a/MC/config/QC/json/pidtof.json +++ b/MC/config/QC/json/pidtof.json @@ -1,72 +1,65 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, - "infologger": { - "": "Configuration of the Infologger (optional).", - "filterDiscardDebug": "false", - "": "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel": "21", - "": "Message at this level or above are discarded (default: 21 - Trace)" - } - }, - "tasks": { - "PID": { - "active": "true", - "className": "o2::quality_control_modules::pid::TaskFT0TOF", - "moduleName": "QcTOF", - "detectorName": "TOF", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query_comment": "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0" - }, - "taskParameters": { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", - "verbose": "false", - "minPtCut": "0.3f", - "etaCut": "0.8f", - "minNTPCClustersCut": "60", - "minDCACut": "100.f", - "minDCACutY": "10.f" - }, - "grpGeomRequest" : { - "geomRequest": "Aligned", - "askGRPECS": "false", - "askGRPLHCIF": "false", - "askGRPMagField": "true", - "askMatLUT": "false", - "askTime": "false", - "askOnceAllButField": "true", - "needPropagatorD": "false" - }, - "": "For debugging, path to the file where to save. If empty or missing it won't save." - } - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "dataSamplingPolicies": [] -} + "tasks": { + "PID": { + "active": "true", + "className": "o2::quality_control_modules::pid::TaskFT0TOF", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0" + }, + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", + "verbose": "false", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + }, + "": "For debugging, path to the file where to save. If empty or missing it won't save." + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/pidtofNoTRD.json b/MC/config/QC/json/pidtofNoTRD.json index 6b5781a6e..f249889c1 100644 --- a/MC/config/QC/json/pidtofNoTRD.json +++ b/MC/config/QC/json/pidtofNoTRD.json @@ -1,72 +1,65 @@ { - "qc": { - "config": { - "database": { - "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, - "Activity": { - "number": "42", - "type": "2", - "provenance": "qc_mc", - "passName": "passMC", - "periodName": "SimChallenge" - }, - "monitoring": { - "url": "no-op://" - }, - "consul": { - "url": "" - }, - "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, - "infologger": { - "": "Configuration of the Infologger (optional).", - "filterDiscardDebug": "false", - "": "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel": "21", - "": "Message at this level or above are discarded (default: 21 - Trace)" - } - }, - "tasks": { - "PID": { - "active": "true", - "className": "o2::quality_control_modules::pid::TaskFT0TOF", - "moduleName": "QcTOF", - "detectorName": "TOF", - "cycleDurationSeconds": "10", - "maxNumberCycles": "-1", - "dataSource": { - "type": "direct", - "query_comment": "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC" - }, - "taskParameters": { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", - "verbose": "false", - "minPtCut": "0.3f", - "etaCut": "0.8f", - "minNTPCClustersCut": "60", - "minDCACut": "100.f", - "minDCACutY": "10.f" - }, - "grpGeomRequest" : { - "geomRequest": "Aligned", - "askGRPECS": "false", - "askGRPLHCIF": "false", - "askGRPMagField": "true", - "askMatLUT": "false", - "askTime": "false", - "askOnceAllButField": "true", - "needPropagatorD": "false" - }, - "": "For debugging, path to the file where to save. If empty or missing it won't save." - } - } + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "dataSamplingPolicies": [] -} + "tasks": { + "PID": { + "active": "true", + "className": "o2::quality_control_modules::pid::TaskFT0TOF", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC" + }, + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", + "verbose": "false", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" + }, + "grpGeomRequest": { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "false", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + }, + "": "For debugging, path to the file where to save. If empty or missing it won't save." + } + } + }, + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/tof-trending-hits.json b/MC/config/QC/json/tof-trending-hits.json index 81126e8f1..7b8ccbc34 100644 --- a/MC/config/QC/json/tof-trending-hits.json +++ b/MC/config/QC/json/tof-trending-hits.json @@ -9,10 +9,11 @@ "name": "not_applicable" }, "Activity": { - "number": "", + "number": "42", + "type": "2", + "provenance": "qc_mc", "passName": "passMC", - "periodName": "", - "provenance" : "qc_mc" + "periodName": "SimChallenge" }, "monitoring": { "url": "no-op://" @@ -21,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "postprocessing": { @@ -62,5 +63,4 @@ } } } -} - +} \ No newline at end of file diff --git a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json index 8df5f9b23..e09190776 100644 --- a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json @@ -1,59 +1,54 @@ { - "qc" : { - "config" : { - "database" : { - "implementation" : "CCDB", - "host" : "ccdb-test.cern.ch:8080", - "username" : "not_applicable", - "password" : "not_applicable", - "name" : "not_applicable" + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "Activity" : { - "number" : "42", - "type" : "2", - "provenance" : "qc_mc", - "passName" : "passMC", - "periodName" : "SimChallenge" + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" }, - "monitoring" : { - "url" : "no-op://" + "monitoring": { + "url": "no-op://" }, - "consul" : { - "url" : "" + "consul": { + "url": "" }, - "conditionDB" : { - "url" : "ccdb-test.cern.ch:8080" - }, - "infologger" : { "" : "Configuration of the Infologger (optional).", - "filterDiscardDebug" : "false", - "" : "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel" : "21", - "" : "Message at this level or above are discarded (default: 21 - Trace)" } + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "tasks" : { - "MatchTrAll" : { - "active" : "true", - "className" : "o2::quality_control_modules::tof::TOFMatchedTracks", - "moduleName" : "QcTOF", - "detectorName" : "TOF", - "cycleDurationSeconds" : "10", - "maxNumberCycles" : "-1", - "dataSource" : { - "type" : "direct", - "query_comment" : "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0" + "tasks": { + "MatchTrAll": { + "active": "true", + "className": "o2::quality_control_modules::tof::TOFMatchedTracks", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC;trackITSTPCTRD:TRD/MATCH_ITSTPC/0;trackITSTPCTRDMCTR:TRD/MCLB_ITSTPC/0;trackITSTPCTRDSAMCTR:TRD/MCLB_ITSTPC_TRD/0;trackTPCTRD:TRD/MATCH_TPC/0;trackTPCTRDMCTR:TRD/MCLB_TPC/0;trackTPCTRDSAMCTR:TRD/MCLB_TPC_TRD/0;trigITSTPCTRD:TRD/TRGREC_ITSTPC/0;trigTPCTRD:TRD/TRGREC_TPC/0;matchITSTPCTRDTOF:TOF/MTC_ITSTPCTRD/0;clsTOF_GLO3_MCTR:TOF/MCMTC_ITSTPCTRD/0;matchTPCTRDTOF:TOF/MTC_TPCTRD/0;clsTOF_GLO2_MCTR:TOF/MCMTC_TPCTRD/0" }, - "taskParameters" : { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", - "verbose" : "false", - "isMC" : "true", - "minPtCut" : "0.3f", - "etaCut" : "0.8f", - "minNTPCClustersCut" : "60", - "minDCACut" : "100.f", - "minDCACutY" : "10.f" + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", + "verbose": "false", + "isMC": "true", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -61,13 +56,13 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" }, - "location" : "remote", - "saveObjectsToFile" : "TOFmatchedTracks_AllTypes_MC.root", - "" : "For debugging, path to the file where to save. If empty or missing it won't save." + "location": "remote", + "saveObjectsToFile": "TOFmatchedTracks_AllTypes_MC.root", + "": "For debugging, path to the file where to save. If empty or missing it won't save." } } }, - "dataSamplingPolicies" : [] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json index 3a64ca2ca..34f330b28 100644 --- a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json @@ -1,60 +1,55 @@ { - "qc" : { - "config" : { - "database" : { - "implementation" : "CCDB", - "host" : "ccdb-test.cern.ch:8080", - "username" : "not_applicable", - "password" : "not_applicable", - "name" : "not_applicable" + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "Activity" : { - "number" : "42", - "type" : "2", - "provenance" : "qc_mc", - "passName" : "passMC", - "periodName" : "SimChallenge" + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" }, - "monitoring" : { - "url" : "no-op://" + "monitoring": { + "url": "no-op://" }, - "consul" : { - "url" : "" + "consul": { + "url": "" }, - "conditionDB" : { - "url" : "ccdb-test.cern.ch:8080" - }, - "infologger" : { "" : "Configuration of the Infologger (optional).", - "filterDiscardDebug" : "false", - "" : "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel" : "21", - "" : "Message at this level or above are discarded (default: 21 - Trace)" } + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "tasks" : { - "MatchTrNoTRD" : { - "taskName" : "MatchTrNoTRD", - "active" : "true", - "className" : "o2::quality_control_modules::tof::TOFMatchedTracks", - "moduleName" : "QcTOF", - "detectorName" : "TOF", - "cycleDurationSeconds" : "10", - "maxNumberCycles" : "-1", - "dataSource" : { - "type" : "direct", - "query_comment" : "checking every matched track", - "query" : "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC" + "tasks": { + "MatchTrNoTRD": { + "taskName": "MatchTrNoTRD", + "active": "true", + "className": "o2::quality_control_modules::tof::TOFMatchedTracks", + "moduleName": "QcTOF", + "detectorName": "TOF", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "dataSource": { + "type": "direct", + "query_comment": "checking every matched track", + "query": "matchITSTPCTOF:TOF/MTC_ITSTPC/0;matchTPCTOF:TOF/MTC_TPC/0;trackTPCTOF:TOF/TOFTRACKS_TPC/0;trackITSTPC:GLO/TPCITS/0;trackITSTPCABREFS:GLO/TPCITSAB_REFS/0;trackITSTPCABCLID:GLO/TPCITSAB_CLID/0;trackTPC:TPC/TRACKS/0;trackTPCClRefs:TPC/CLUSREFS/0;tofcluster:TOF/CLUSTERS/0;trackTPCMCTR:TPC/TRACKSMCLBL;trackITSTPCMCTR:GLO/TPCITS_MC;trackITSTPCABMCTR:GLO/TPCITSAB_MC;clsTOF_TPC_MCTR:TOF/MCMTC_TPC;clsTOF_GLO_MCTR:TOF/MCMTC_ITSTPC" }, - "taskParameters" : { - "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", - "verbose" : "false", - "isMC" : "true", - "minPtCut" : "0.3f", - "etaCut" : "0.8f", - "minNTPCClustersCut" : "60", - "minDCACut" : "100.f", - "minDCACutY" : "10.f" + "taskParameters": { + "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", + "verbose": "false", + "isMC": "true", + "minPtCut": "0.3f", + "etaCut": "0.8f", + "minNTPCClustersCut": "60", + "minDCACut": "100.f", + "minDCACutY": "10.f" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -62,13 +57,13 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" }, - "location" : "remote", - "saveObjectsToFile" : "TOFmatchedITSTPCTOF_TPCTOF_MC.root", - "" : "For debugging, path to the file where to save. If empty or missing it won't save." + "location": "remote", + "saveObjectsToFile": "TOFmatchedITSTPCTOF_TPCTOF_MC.root", + "": "For debugging, path to the file where to save. If empty or missing it won't save." } } }, - "dataSamplingPolicies" : [] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/tofdigits.json b/MC/config/QC/json/tofdigits.json index 0dc23d3ef..451c930bb 100644 --- a/MC/config/QC/json/tofdigits.json +++ b/MC/config/QC/json/tofdigits.json @@ -11,7 +11,9 @@ "Activity": { "number": "42", "type": "2", - "provenance" : "qc_mc" + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" }, "monitoring": { "url": "no-op://" @@ -20,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -43,7 +45,7 @@ "NbinsMultiplicity": "15000", "applyCalib": "True" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "Aligned", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -51,7 +53,7 @@ "askMatLUT": "false", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" }, "location": "remote" } @@ -139,4 +141,4 @@ "blocking": "false" } ] -} +} \ No newline at end of file diff --git a/MC/config/QC/json/tpc-qc-standard-direct.json b/MC/config/QC/json/tpc-qc-standard-direct.json index c3d3b478e..a1ab6692a 100644 --- a/MC/config/QC/json/tpc-qc-standard-direct.json +++ b/MC/config/QC/json/tpc-qc-standard-direct.json @@ -3,7 +3,10 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080" + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, "Activity": { "number": "42", @@ -12,10 +15,6 @@ "passName": "passMC", "periodName": "SimChallenge" }, - "infologger": { - "filterDiscardDebug": "true", - "filterDiscardLevel": "1" - }, "monitoring": { "url": "no-op://" }, @@ -39,12 +38,24 @@ }, "taskParameters": { "mergeableOutput": "true", - "NClustersNBins": "100", "NClustersXMin": "0", "NClustersXMax": "100", - "QmaxNBins": "200", "QmaxXMin": "0", "QmaxXMax": "200", - "QtotNBins": "600", "QtotXMin": "10", "QtotXMax": "600", - "SigmaPadNBins": "200", "SigmaPadXMin": "0", "SigmaPadXMax": "2", - "SigmaTimeNBins": "200", "SigmaTimeXMin": "0", "SigmaTimeXMax": "2", - "TimeBinNBins": "1000", "TimeBinXMin": "0", "TimeBinXMax": "100000" + "NClustersNBins": "100", + "NClustersXMin": "0", + "NClustersXMax": "100", + "QmaxNBins": "200", + "QmaxXMin": "0", + "QmaxXMax": "200", + "QtotNBins": "600", + "QtotXMin": "10", + "QtotXMax": "600", + "SigmaPadNBins": "200", + "SigmaPadXMin": "0", + "SigmaPadXMax": "2", + "SigmaTimeNBins": "200", + "SigmaTimeXMin": "0", + "SigmaTimeXMax": "2", + "TimeBinNBins": "1000", + "TimeBinXMin": "0", + "TimeBinXMax": "100000" } }, "PID": { @@ -55,7 +66,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query" : "inputTracks:TPC/TRACKS/0" + "query": "inputTracks:TPC/TRACKS/0" }, "taskParameters": { "cutMinNCluster": "60", @@ -66,7 +77,7 @@ "cutMaxpTPC": "20.", "cutMinpTPCMIPs": "0.45", "cutMaxpTPCMIPs": "0.55" - } + } }, "Tracks": { "active": "true", @@ -77,7 +88,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query" : "inputTracks:TPC/TRACKS/0" + "query": "inputTracks:TPC/TRACKS/0" }, "taskParameters": { "cutAbsEta": "1.", @@ -86,7 +97,7 @@ "samplingFractionDCAr": "0.1", "cutPtForDCAr": "1.5" }, - "grpGeomRequest" : { + "grpGeomRequest": { "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", @@ -94,11 +105,10 @@ "askMatLUT": "true", "askTime": "false", "askOnceAllButField": "true", - "needPropagatorD": "false" + "needPropagatorD": "false" } } } }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/tpc-qc-tracking-direct.json b/MC/config/QC/json/tpc-qc-tracking-direct.json index ae98df0cf..bb22d4d60 100644 --- a/MC/config/QC/json/tpc-qc-tracking-direct.json +++ b/MC/config/QC/json/tpc-qc-tracking-direct.json @@ -22,7 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -34,8 +34,8 @@ "cycleDurationSeconds": "10", "maxNumberCycles": "-1", "dataSource": { - "type": "direct", - "query": "inputTracks:TPC/TRACKS/0;inputTrackLabels:TPC/TRACKSMCLBL/0;inputClusRefs:TPC/CLUSREFS/0;inputClusters:TPC/CLUSTERNATIVE/0;inputClusterLabels:TPC/CLNATIVEMCLBL/0" + "type": "direct", + "query": "inputTracks:TPC/TRACKS/0;inputTrackLabels:TPC/TRACKSMCLBL/0;inputClusRefs:TPC/CLUSREFS/0;inputClusters:TPC/CLUSTERNATIVE/0;inputClusterLabels:TPC/CLNATIVEMCLBL/0" }, "taskParameters": { "myOwnKey": "myOwnValue" @@ -43,9 +43,7 @@ "location": "remote" } }, - "checks": { - } + "checks": {} }, - "dataSamplingPolicies": [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/MC/config/QC/json/trd-digits-task.json b/MC/config/QC/json/trd-digits-task.json index 601089072..6b57a85cb 100644 --- a/MC/config/QC/json/trd-digits-task.json +++ b/MC/config/QC/json/trd-digits-task.json @@ -22,11 +22,7 @@ "url": "" }, "conditionDB": { - "url": "ccdb-test.cern.ch:8080" - }, - "infologger": { - "filterDiscardDebug": "false", - "filterDiscardLevel": "2" + "url": "alice-ccdb.cern.ch" } }, "tasks": { @@ -49,7 +45,6 @@ } } }, - "dataSamplingPolicies": [ - ] -} -} + "dataSamplingPolicies": [] + } +} \ No newline at end of file diff --git a/MC/config/QC/json/vertexing-qc-direct-mc.json b/MC/config/QC/json/vertexing-qc-direct-mc.json index 2447b0995..5e83bd3d2 100644 --- a/MC/config/QC/json/vertexing-qc-direct-mc.json +++ b/MC/config/QC/json/vertexing-qc-direct-mc.json @@ -1,72 +1,69 @@ { - "qc" : { - "config" : { - "database" : { - "implementation" : "CCDB", - "host" : "ccdb-test.cern.ch:8080", - "username" : "not_applicable", - "password" : "not_applicable", - "name" : "not_applicable" + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" }, - "Activity" : { - "number" : "42", - "type" : "2", + "Activity": { + "number": "42", + "type": "2", "provenance": "qc_mc", "passName": "passMC", "periodName": "SimChallenge" }, - "monitoring" : { - "url" : "no-op://" + "monitoring": { + "url": "no-op://" }, - "consul" : { - "url" : "" + "consul": { + "url": "" }, - "conditionDB" : { - "url" : "ccdb-test.cern.ch:8080" - }, - "infologger" : { "" : "Configuration of the Infologger (optional).", - "filterDiscardDebug" : "1", - "" : "Set to true to discard debug and trace messages (default: false)", - "filterDiscardLevel" : "21", - "" : "Message at this level or above are discarded (default: 21 - Trace)" } + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } }, - "tasks" : { - "Vertexing" : { - "active" : "true", - "className" : "o2::quality_control_modules::glo::VertexingQcTask", - "moduleName" : "QcGLO", - "detectorName" : "GLO", - "cycleDurationSeconds" : "10", - "maxNumberCycles" : "-1", - "" : "The other type of dataSource is \"direct\", see basic-no-sampling.json.", - "dataSource" : { - "type" : "direct", - "query_comment" : "checking every vertex, in MC", - "query" : "pvtx:GLO/PVTX/0;pvtxLbl:GLO/PVTX_MCTR/0" + "tasks": { + "Vertexing": { + "active": "true", + "className": "o2::quality_control_modules::glo::VertexingQcTask", + "moduleName": "QcGLO", + "detectorName": "GLO", + "cycleDurationSeconds": "10", + "maxNumberCycles": "-1", + "": "For debugging, path to the file where to save. If empty or missing it won't save.", + "dataSource": { + "type": "direct", + "query_comment": "checking every vertex, in MC", + "query": "pvtx:GLO/PVTX/0;pvtxLbl:GLO/PVTX_MCTR/0" }, - "taskParameters" : { - "isMC" : "true" + "taskParameters": { + "isMC": "true" }, - "location" : "remote", - "saveObjectsToFile" : "testVertexingQC_MC.root", - "" : "For debugging, path to the file where to save. If empty or missing it won't save." + "location": "remote", + "saveObjectsToFile": "testVertexingQC_MC.root" } }, - "checks" : { - "QcCheck" : { - "active" : "false", - "className" : "o2::quality_control_modules::skeleton::SkeletonCheck", - "moduleName" : "QcSkeleton", - "policy" : "OnAny", - "detectorName" : "GLO", - "dataSource" : [ { - "type" : "Task", - "name" : "Vertexing", - "MOs" : ["example"] - } ] + "checks": { + "QcCheck": { + "active": "false", + "className": "o2::quality_control_modules::skeleton::SkeletonCheck", + "moduleName": "QcSkeleton", + "policy": "OnAny", + "detectorName": "GLO", + "dataSource": [ + { + "type": "Task", + "name": "Vertexing", + "MOs": [ + "example" + ] + } + ] } } }, - "dataSamplingPolicies" : [ - ] -} + "dataSamplingPolicies": [] +} \ No newline at end of file diff --git a/UTILS/update-qc-json.py b/UTILS/update-qc-json.py new file mode 100644 index 000000000..4796756e9 --- /dev/null +++ b/UTILS/update-qc-json.py @@ -0,0 +1,62 @@ +import os +import json +import argparse + +# Embedded template qc configuration for MC +template_data = { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + } +} + +def update_json_files(folder_path): + # Iterate over files in the specified folder + for filename in os.listdir(folder_path): + if filename.endswith('.json'): + file_path = os.path.join(folder_path, filename) + + # Read the content of the JSON file + with open(file_path, 'r') as file: + data = json.load(file) + + # Check if 'qc' and 'config' sections exist and then update + if 'qc' in data and 'config' in data['qc']: + data['qc']['config'] = template_data['config'] + + # Write the updated content back to the JSON file + with open(file_path, 'w') as file: + json.dump(data, file, indent=2) + +def main(): + parser = argparse.ArgumentParser(description="Update the 'config' section in the 'qc' part of JSON files in a folder.") + parser.add_argument('folder_path', type=str, help='Path to the folder containing JSON files') + + args = parser.parse_args() + + update_json_files(args.folder_path) + +if __name__ == "__main__": + main() + From e8a720d3e972f2e73d7d2cdacdb540db3f6c2109 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 20 Dec 2023 09:24:49 +0100 Subject: [PATCH 0295/1239] Avoid checking O2trackqa table, which is downsampled and won't pass the sanity check --- DATA/production/common/readAO2Ds.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/common/readAO2Ds.C b/DATA/production/common/readAO2Ds.C index 7b5c9c84b..2574b8230 100644 --- a/DATA/production/common/readAO2Ds.C +++ b/DATA/production/common/readAO2Ds.C @@ -35,7 +35,7 @@ int readAO2Ds(const char* filename = "AO2D.root") std::cout << onameKeyInDir.Data() << std::endl; } TTree* t = (TTree*)d->Get(onameKeyInDir.Data()); - if (onameKeyInDir.BeginsWith("O2track") && !onameKeyInDir.Contains("O2tracked")) { + if (onameKeyInDir.BeginsWith("O2track") && !onameKeyInDir.Contains("O2tracked") && !onameKeyInDir.Contains("O2trackqa")) { vectNEntriesPerTree.push_back({onameKeyInDir.Data(), t->GetEntries()}); } } From 66a6be1a071c56e495e4a27f832069c434153b52 Mon Sep 17 00:00:00 2001 From: shahoian Date: Thu, 21 Dec 2023 00:57:01 +0100 Subject: [PATCH 0296/1239] Decrease safety time margin for ITSTPC matching to 2ms --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 0b37841f5..0818182e9 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -282,7 +282,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then [[ -z $TPCITSTIMEERR ]] && TPCITSTIMEERR="0.2" [[ -z $ITS_CONFIG || "$ITS_CONFIG" != *"--tracking-mode"* ]] && export ITS_CONFIG+=" --tracking-mode async" CUT_MATCH_CHI2=160 - export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=10.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=5;tpcitsMatch.crudeNSigma2Cut[0]=100;tpcitsMatch.crudeNSigma2Cut[1]=100;tpcitsMatch.crudeNSigma2Cut[2]=100;tpcitsMatch.crudeNSigma2Cut[3]=100;tpcitsMatch.crudeNSigma2Cut[4]=100;" + export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=5;tpcitsMatch.crudeNSigma2Cut[0]=100;tpcitsMatch.crudeNSigma2Cut[1]=100;tpcitsMatch.crudeNSigma2Cut[2]=100;tpcitsMatch.crudeNSigma2Cut[3]=100;tpcitsMatch.crudeNSigma2Cut[4]=100;" # enabling TPC calibration scaling # the default is to use CTP, unless specified differently in the JDL... From 16c4587b3d8295e9c9bf991333948c2312ff1848 Mon Sep 17 00:00:00 2001 From: Fabio Catalano Date: Thu, 21 Dec 2023 09:12:55 +0100 Subject: [PATCH 0297/1239] Add bash script for D2H anchored MC --- ..._pp_anchor2022_D2H_ccbar_and_bbbar_gap5.sh | 180 ++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100755 MC/run/PWGHF/run_pp_anchor2022_D2H_ccbar_and_bbbar_gap5.sh diff --git a/MC/run/PWGHF/run_pp_anchor2022_D2H_ccbar_and_bbbar_gap5.sh b/MC/run/PWGHF/run_pp_anchor2022_D2H_ccbar_and_bbbar_gap5.sh new file mode 100755 index 000000000..3dd5f2902 --- /dev/null +++ b/MC/run/PWGHF/run_pp_anchor2022_D2H_ccbar_and_bbbar_gap5.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# add distortion maps +# https://alice.its.cern.ch/jira/browse/O2-3346?focusedCommentId=300982&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-300982 +# +# export O2DPG_ENABLE_TPC_DISTORTIONS=ON +# SCFile=$PWD/distortions_5kG_lowIR.root # file needs to be downloaded +# export O2DPG_TPC_DIGIT_EXTRA=" --distortionType 2 --readSpaceCharge ${SCFile} " + +# +# procedure setting up and executing an anchored MC +# + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ------ CREATE AN MC CONFIG STARTING FROM RECO SCRIPT -------- +# - this part should not be done on the GRID, where we should rather +# point to an existing config (O2DPG repo or local disc or whatever) +export ALIEN_JDL_LPMANCHORYEAR=${ALIEN_JDL_LPMANCHORYEAR:-2022} +RUNNUMBER=${ALIEN_JDL_LPMRUNNUMBER:-517616} +#INTERACTIONRATE=${INTERACTIONRATE:-2000} + +# get the async script (we need to modify it) +# the script location can be configured with a JDL option + +ALIEN_JDL_LPMPRODUCTIONTAG_KEEP=$ALIEN_JDL_LPMPRODUCTIONTAG +echo "Substituting ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG with ALIEN_JDL_LPMANCHORPRODUCTION=$ALIEN_JDL_LPMANCHORPRODUCTION for simulating reco pass..." +ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMANCHORPRODUCTION + +# ZDC causes issues for sim +#export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,ZDC,CTP +export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP + +### async_pass.sh +DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh + +if [[ -f async_pass.sh ]]; then + chmod +x async_pass.sh + DPGRECO=./async_pass.sh +else + cp -v $DPGRECO . +fi + +if [[ ! -f setenv_extra.sh ]]; then + cp -v ${DPGRECO%/*}/setenv_extra.sh . +fi + +echo "[INFO alien_async_pass.sh] Setting up DPGRECO to ${DPGRECO}" + +#settings that are MC-specific +sed -i 's/GPU_global.dEdxUseFullGainMap=1;GPU_global.dEdxDisableResidualGainMap=1/GPU_global.dEdxSplineTopologyCorrFile=splines_for_dedx_V1_MC_iter0_PP.root;GPU_global.dEdxDisableTopologyPol=1;GPU_global.dEdxDisableGainMap=1;GPU_global.dEdxDisableResidualGainMap=1;GPU_global.dEdxDisableResidualGain=1/' setenv_extra.sh +### ??? + +chmod +x async_pass.sh + +# take out line running the workflow (if we don't have data input) +[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh + +# create workflow ---> creates the file that can be parsed +export IGNORE_EXISTING_SHMFILES=1 +touch list.list + +./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log +RECO_RC=$? + +echo "RECO finished with ${RECO_RC}" +if [ "${NO_MC}" ]; then + return ${RECO_RC} 2>/dev/null || exit ${RECO_RC} # optionally quit here and don't do MC (useful for testing) +fi + +ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP +echo "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG" + +# now create the local MC config file --> config-config.json +${O2DPG_ROOT}/UTILS/parse-async-WorkflowConfig.py + +# check if config reasonably created +if [[ `grep "o2-ctf-reader-workflow-options" config-json.json 2> /dev/null | wc -l` == "0" ]]; then + echo "Problem in anchor config creation. Stopping." + exit 1 +fi + +# -- CREATE THE MC JOB DESCRIPTION ANCHORED TO RUN -- + +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" +SIMENGINE=${SIMENGINE:-TGeant4} +SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE}} +NTIMEFRAMES=${NTIMEFRAMES:-50} +NSIGEVENTS=${NSIGEVENTS:-22} + +SPLITID=${SPLITID:-0} +PRODSPLIT=${PRODSPLIT:-100} +CYCLE=${CYCLE:-0} +# let SEED=$SPLITID+$CYCLE*$PRODSPLIT +SEED=${ALIEN_PROC_ID} + +# create workflow +# THIS NEEDS TO COME FROM OUTSIDE +# echo "$" | awk -F' -- ' '{print $1, $3}' + +# baseargs="-col pp -eCM 13600 -tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${RUNNUMBER}" +baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${RUNNUMBER}" + +# THIS NEEDS TO COME FROM OUTSIDE +remainingargs="-gen external -ini $O2DPG_ROOT/MC/config/PWGHF/ini/GeneratorHF_D2H_ccbar_and_bbbar_gap5.ini -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc" + +remainingargs="${remainingargs} -e ${SIMENGINE} -j ${NWORKERS}" +remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" +remainingargs="${remainingargs} --anchor-config config-json.json" + +echo "baseargs: ${baseargs}" +echo "remainingargs: ${remainingargs}" + +# query CCDB has changed, w/o "_" +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> timestampsampling_${RUNNUMBER}.log +[ "$?" != "0" ] && echo "Problem during anchor timestamp sampling " && exit 1 + +TIMESTAMP=`grep "Determined timestamp to be" timestampsampling_${RUNNUMBER}.log | awk '//{print $6}'` +echo "TIMESTAMP IS ${TIMESTAMP}" + +# -- PREFETCH CCDB OBJECTS TO DISC -- +# (make sure the right objects at the right timestamp are fetched +# until https://alice.its.cern.ch/jira/browse/O2-2852 is fixed) +export ALICEO2_CCDB_LOCALCACHE=$PWD/.ccdb +[ ! -d .ccdb ] && mkdir .ccdb + +CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /FT0/Calibration/ChannelTimeOffset /FV0/Calibration/ChannelTimeOffset /GLO/GRP/BunchFilling" + +${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS} -d .ccdb --timestamp ${TIMESTAMP} +if [ ! "$?" == "0" ]; then + echo "Problem during CCDB prefetching of ${CCDBOBJECTS}. Exiting." + exit 1 +fi + +# -- Create aligned geometry using ITS and MFT ideal alignments to avoid overlaps in geant +CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align MFT/Calib/Align" +TIMESTAMP_IDEAL_MC=1 +${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS_IDEAL_MC} -d .ccdb --timestamp ${TIMESTAMP_IDEAL_MC} +if [ ! "$?" == "0" ]; then + echo "Problem during CCDB prefetching of ${CCDBOBJECTS_IDEAL_MC}. Exiting." + exit 1 +fi + +# ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align,MFT/Calib/Align -b +echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align,MFT/Calib/Align -b +mkdir -p $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned +ln -s -f $PWD/o2sim_geometry-aligned.root $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned/snapshot.root + +# -- RUN THE MC WORKLOAD TO PRODUCE AOD -- + +export FAIRMQ_IPC_PREFIX=./ + +echo "Ready to start main workflow" + +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} +MCRC=$? # <--- we'll report back this code +#exit 0 + +if [[ "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then + # do QC tasks + echo "Doing QC" + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k + RC=$? +fi + +# +# full logs tar-ed for output, regardless the error code or validation - to catch also QC logs... +# +if [[ -n "$ALIEN_PROC_ID" ]]; then + find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "pythia8.cfg" \) | tar -czvf debug_log_archive.tgz -T - + find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T - +fi + +unset FAIRMQ_IPC_PREFIX + +return ${MCRC} 2>/dev/null || exit ${MCRC} + \ No newline at end of file From d1b67818827bdbfe79af41ae4c05df745d33a996 Mon Sep 17 00:00:00 2001 From: rmunzer <97919772+rmunzer@users.noreply.github.com> Date: Thu, 21 Dec 2023 21:24:22 +0100 Subject: [PATCH 0298/1239] TPC - Update calibration and krypton workflows (#1400) * Remove information spec and modify writeout disable parameter in tpc_krypton workflow * Fix for grp in tpc_krypton * Adopt Krypton raw workflow * Change ccdb link and last TimeBin * Fix bug in workflow * Change creation of workflow with scripts * Change workflow creation - using script - for tpc-krypton-raw * Change workflow creation using topo scripts --- DATA/production/calib/tpc-laser-filter.sh | 29 ++++----- DATA/production/calib/tpc-pedestal.sh | 43 +++++++++---- DATA/production/calib/tpc-pulser-long.sh | 43 +++++++++---- DATA/production/calib/tpc-pulser.sh | 43 +++++++++---- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 55 ++++++++++------- DATA/testing/detectors/TPC/tpc-krypton.sh | 61 +++++++++++++------ 6 files changed, 181 insertions(+), 93 deletions(-) diff --git a/DATA/production/calib/tpc-laser-filter.sh b/DATA/production/calib/tpc-laser-filter.sh index a13267b8a..42ba384f8 100755 --- a/DATA/production/calib/tpc-laser-filter.sh +++ b/DATA/production/calib/tpc-laser-filter.sh @@ -10,16 +10,18 @@ FILEWORKDIR="/home/wiechula/processData/inputFilesTracking/triggeredLaser" FILEWORKDIR2="/home/epn/odc/files/" +#ARGS_ALL_CONFIG+="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR;keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null" +ARGS_ALL_CONFIG+="NameConf.mDirGRP=$FILEWORKDIR;NameConf.mDirGeom=$FILEWORKDIR2;NameConf.mDirCollContext=$FILEWORKDIR;NameConf.mDirMatLUT=$FILEWORKDIR" -if [ $NUMAGPUIDS != 0 ]; then +if [ ${NUMAGPUIDS} != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi -if [ $GPUTYPE == "HIP" ]; then - if [ $NUMAID == 0 ] || [ $NUMAGPUIDS == 0 ]; then +if [ ${GPUTYPE} == "HIP" ]; then + if [ ${NUMAID} == 0 ] || [ ${NUMAGPUIDS} == 0 ]; then export TIMESLICEOFFSET=0 else - export TIMESLICEOFFSET=$NGPUS + export TIMESLICEOFFSET=${NGPUS} fi GPU_CONFIG_KEY+="GPU_proc.deviceNum=0;" GPU_CONFIG+=" --environment ROCR_VISIBLE_DEVICES={timeslice${TIMESLICEOFFSET}}" @@ -28,13 +30,13 @@ else GPU_CONFIG_KEY+="GPU_proc.deviceNum=-2;" fi -if [ $GPUTYPE != "CPU" ]; then +if [ ${GPUTYPE} != "CPU" ]; then GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;" - if [ $HOSTMEMSIZE == "0" ]; then + if [ ${HOSTMEMSIZE} == "0" ]; then HOSTMEMSIZE=$(( 1 << 30 )) fi fi -if [ $HOSTMEMSIZE != "0" ]; then +if [ ${HOSTMEMSIZE} != "0" ]; then GPU_CONFIG_KEY+="GPU_proc.forceHostMemoryPoolSize=$HOSTMEMSIZE;" fi @@ -53,30 +55,29 @@ if [[ ! -z ${TPC_LASER_ILBZS:-} ]]; then LASER_DECODER_ADD="--pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root -decode-type 0" fi -o2-dpl-raw-proxy $ARGS_ALL \ +o2-dpl-raw-proxy ${ARGS_ALL} \ --dataspec "$PROXY_INSPEC" --inject-missing-data \ --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL ${LASER_DECODER_ADD} \ + | o2-tpc-raw-to-digits-workflow ${ARGS_ALL} ${LASER_DECODER_ADD} \ --input-spec "$CALIB_INSPEC" \ --configKeyValues "TPCDigitDump.NoiseThreshold=3;TPCDigitDump.LastTimeBin=600;$ARGS_ALL_CONFIG" \ - --pedestal-url /home/wiechula/processData/inputFilesTracking/triggeredLaser/pedestals.openchannels.root \ --pipeline tpc-raw-to-digits-0:20 \ --remove-duplicates \ --send-ce-digits \ - | o2-tpc-reco-workflow $ARGS_ALL ${TPC_CORR_SCALING:-} \ + | o2-tpc-reco-workflow ${ARGS_ALL} ${TPC_CORR_SCALING:-} \ --input-type digitizer \ --output-type "tracks,disable-writer,clusters" \ --disable-mc \ --pipeline tpc-zsEncoder:20,tpc-tracker:8 \ - $GPU_CONFIG \ + ${GPU_CONFIG} \ --condition-remap "file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPECS;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser/=GLO/Config/GRPMagField;file:///home/wiechula/processData/inputFilesTracking/triggeredLaser=TPC/Calib/LaserTracks" \ --configKeyValues "${ARGS_ALL_CONFIG};align-geom.mDetectors=none;GPU_global.deviceType=$GPUTYPE;GPU_proc.tpcIncreasedMinClustersPerRow=500000;GPU_proc.ignoreNonFatalGPUErrors=1;$GPU_CONFIG_KEY;GPU_global.tpcTriggeredMode=1;GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;GPU_rec_tpc.clustersShiftTimebinsClusterizer=35;GPU_proc.memoryScalingFactor=2" \ - | o2-tpc-laser-track-filter $ARGS_ALL \ + | o2-tpc-laser-track-filter ${ARGS_ALL} \ | o2-dpl-output-proxy ${ARGS_ALL} \ --dataspec "$PROXY_OUTSPEC" \ --proxy-name tpc-laser-input-proxy \ --proxy-channel-name tpc-laser-input-proxy \ --channel-config "name=tpc-laser-input-proxy,method=connect,type=push,transport=zeromq,rateLogging=0" \ | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} + | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/production/calib/tpc-pedestal.sh b/DATA/production/calib/tpc-pedestal.sh index 0261c3822..5af914c9d 100755 --- a/DATA/production/calib/tpc-pedestal.sh +++ b/DATA/production/calib/tpc-pedestal.sh @@ -19,7 +19,8 @@ CCDB_PATH="http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn" +#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn" +QC_CONFIG="/o2/components/qc/ANY/any/tpc-pedestal-calib-qcmn" max_events=50 publish_after=400 @@ -37,14 +38,32 @@ EXTRA_CONFIG=" --publish-after-tfs ${publish_after} --max-events ${max_events} - ################################################################################################################################# -o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ - --dataspec "${PROXY_INSPEC}" \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw ${ARGS_ALL} \ - --input-spec "${CALIB_INSPEC}" \ - --configKeyValues "${CALIB_CONFIG}" \ - ${EXTRA_CONFIG} \ - | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ - --ccdb-path ${CCDB_PATH} \ - | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ - | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0 +add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW +fi + +#o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ +# --dataspec "${PROXY_INSPEC}" \ +# --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ +# | o2-tpc-calib-pad-raw ${ARGS_ALL} \ +# --input-spec "${CALIB_INSPEC}" \ +# --configKeyValues "${CALIB_CONFIG}" \ +# ${EXTRA_CONFIG} \ +# | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ +# --ccdb-path ${CCDB_PATH} \ +# | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ +# | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/production/calib/tpc-pulser-long.sh b/DATA/production/calib/tpc-pulser-long.sh index 8cd2c38e9..e1aaab4d4 100755 --- a/DATA/production/calib/tpc-pulser-long.sh +++ b/DATA/production/calib/tpc-pulser-long.sh @@ -18,8 +18,8 @@ CCDB_PATH="http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" - +#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" +QC_CONFIG="/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn" max_events=1000000 publish_after=200 @@ -35,15 +35,32 @@ EXTRA_CONFIG="--calib-type pulser --reset-after-publish --publish-after-tfs ${pu ################################################################################################################################# +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --calib-type pulser --reset-after-publish --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0 +add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW +fi + -o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ - --dataspec ${PROXY_INSPEC} \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw ${ARGS_ALL} \ - --input-spec ${CALIB_INSPEC} \ - --configKeyValues "${CALIB_CONFIG}" \ - ${EXTRA_CONFIG} \ - | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ - --ccdb-path ${CCDB_PATH} \ - | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ - | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} +#o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ +# --dataspec ${PROXY_INSPEC} \ +# --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ +# | o2-tpc-calib-pad-raw ${ARGS_ALL} \ +# --input-spec ${CALIB_INSPEC} \ +# --configKeyValues "${CALIB_CONFIG}" \ +# ${EXTRA_CONFIG} \ +# | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ +# --ccdb-path ${CCDB_PATH} \ +# | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ +# | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index b3e9bde67..6b68a030b 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -20,7 +20,8 @@ CCDB_PATH="http://o2-ccdb.internal" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn" +#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn" +QC_CONFIG="/o2/components/qc/ANY/any/tpc-pulser-calib-qcmn" max_events=200 publish_after=230 @@ -38,14 +39,32 @@ EXTRA_CONFIG="--calib-type pulser --publish-after-tfs ${publish_after} --max-eve ################################################################################################################################# -o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ - --dataspec "${PROXY_INSPEC}" \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-calib-pad-raw ${ARGS_ALL} \ - --input-spec "${CALIB_INSPEC}" \ - --configKeyValues "${CALIB_CONFIG}" \ - ${EXTRA_CONFIG} \ - | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ - --ccdb-path ${CCDB_PATH} \ - | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ - | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-tpc-calib-pad-raw "--input-spec \"$CALIB_INSPEC\" --calib-type pulser --publish-after-tfs ${publish_after} --max-events ${max_events} --lanes 36 --check-calib-infos" "${CALIB_CONFIG}" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path \"http://o2-ccdb.internal\" " "" 0 +add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW +fi + +#o2-dpl-raw-proxy ${ARGS_ALL} --inject-missing-data \ +# --dataspec "${PROXY_INSPEC}" \ +# --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ +# | o2-tpc-calib-pad-raw ${ARGS_ALL} \ +# --input-spec "${CALIB_INSPEC}" \ +# --configKeyValues "${CALIB_CONFIG}" \ +# ${EXTRA_CONFIG} \ +# | o2-calibration-ccdb-populator-workflow ${ARGS_ALL} \ +# --ccdb-path ${CCDB_PATH} \ +# | o2-qc ${ARGS_ALL} --config ${QC_CONFIG} --local --host ${HOST} \ +# | o2-dpl-run ${ARGS_ALL} --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index bbdfa743b..f62b09d57 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -7,34 +7,43 @@ source common/gen_topo_helper_functions.sh export SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" NLANES=36 SESSION="default" ARGS_FILES="keyval.output_dir=/dev/null" HOST=localhost -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-raw-qcmn" + +#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" +QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" + + + +WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --max-tf-per-file 8000" + +if [[ ! -z ${TPC_KRYPTON_NO_WRITEOUT:-} ]]; then + WRITER_TYPE="--writer-type none" +fi + + # TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow # as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" --inject-missing-data \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$ARGS_FILES" \ - --remove-duplicates \ - --pipeline tpc-raw-to-digits-0:24 \ - --pedestal-url "http://ccdb-test.cern.ch:8080" \ - | o2-tpc-krypton-raw-filter $ARGS_ALL \ - --configKeyValues "$ARGS_FILES" \ - --lanes $NLANES \ - --writer-type EPN \ - --meta-output-dir /data/epn2eos_tool/epn2eos/ \ - --output-dir /data/tf/raw \ - --threshold-max 20 \ - --max-tf-per-file 8000 \ - --time-bins-before 20 \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host $HOST \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pedestal-url \"http://o2-ccdb.internal\" --pipeline tpc-raw-to-digits-0:24 " "\"${ARGS_FILES}\";TPCDigitDump.LastTimeBin=446" +add_W o2-tpc-krypton-raw-filter "${WRITER_TYPE} --lanes $NLANES --threshold-max 20 --time-bins-before 20" "\"${ARGS_FILES}\"" +add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW +fi + diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index d7dc474eb..4d987d602 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -13,8 +13,8 @@ if [ $NUMAGPUIDS != 0 ]; then ARGS_ALL+=" --child-driver 'numactl --membind $NUMAID --cpunodebind $NUMAID'" fi -PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" -CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" +CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" NLANES=1 @@ -22,28 +22,51 @@ SESSION="default" ARGS_FILES="keyval.output_dir=/dev/null" -QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" +#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" +QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-qcmn" WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR" -if [[ ${TPC_KRYPTON_NO_WRITEOUT:-} == 1 ]]; then + +if [[ ! -z ${TPC_KRYPTON_NO_WRITEOUT:-} ]]; then WRITER_TYPE="--writer-type none" fi # TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow # as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" --inject-missing-data \ - --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ - | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ - --input-spec "$CALIB_INSPEC" \ - --configKeyValues "$ARGS_FILES" \ - --remove-duplicates \ - --pipeline tpc-raw-to-digits-0:20 \ - | o2-tpc-krypton-clusterer $ARGS_ALL \ - ${WRITER_TYPE} \ - --lanes $NLANES \ - --configKeyValues "$ARGS_FILES" \ - --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ - | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host localhost \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} + + +# Add binarry to workflow command USAGE: add_W [BINARY] [COMMAND_LINE_OPTIONS] [CONFIG_KEY_VALUES] [Add ARGS_ALL_CONFIG, optional, default = 1] + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pipeline tpc-raw-to-digits-0:20 " "\"${ARGS_FILES}\";TPCDigitDump.LastTimeBin=14256" +add_W o2-tpc-krypton-clusterer "${WRITER_TYPE} --lanes $NLANES --configFile=\"/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini\"" "\"${ARGS_FILES}\"" +add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW +fi + +##o2-dpl-raw-proxy $ARGS_ALL \ +# --dataspec "$PROXY_INSPEC" --inject-missing-data \ +# --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ +# | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ +# --ignore-grp \ +# --input-spec "$CALIB_INSPEC" \ +# --configKeyValues "$ARGS_FILES;TPCDigitDump.LastTimeBin=14256" \ +# --remove-duplicates \ +# --pipeline tpc-raw-to-digits-0:20 \ +# | o2-tpc-krypton-clusterer $ARGS_ALL \ +# ${WRITER_TYPE} \ +# --lanes $NLANES \ +# --configKeyValues "$ARGS_FILES" \ +## --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ +# | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host localhost \ +# | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} From cab4fcc2ed2304ea7ee82994aa45aec01c1177d0 Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Thu, 21 Dec 2023 23:32:18 +0100 Subject: [PATCH 0299/1239] Fix max. ITS chi2 cut. --- DATA/production/qc-async/itstpc.json | 2 +- MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/itstpc.json b/DATA/production/qc-async/itstpc.json index 50ad83d1b..5bce54d2e 100644 --- a/DATA/production/qc-async/itstpc.json +++ b/DATA/production/qc-async/itstpc.json @@ -47,7 +47,7 @@ "minPtITSCut": "0.f", "etaITSCut": "1e10f", "minNITSClustersCut": "0", - "maxChi2PerClusterITS": "1e10f", + "maxChi2PerClusterITS": "100000", "minPtTPCCut": "0.1f", "etaTPCCut": "1.4f", "minNTPCClustersCut": "60", diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index 95d25f655..f2a9ead84 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -44,7 +44,7 @@ "minPtITSCut": "0.f", "etaITSCut": "1e10f", "minNITSClustersCut": "0", - "maxChi2PerClusterITS": "1e10f", + "maxChi2PerClusterITS": "10000", "minPtTPCCut": "0.1f", "etaTPCCut": "1.4f", "minNTPCClustersCut": "60", From 3bb1faa29bdc331da07d233b3d0d3ee7e52323d2 Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Thu, 21 Dec 2023 23:33:27 +0100 Subject: [PATCH 0300/1239] Update max. ITS chi2 cut in MC. --- MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index f2a9ead84..a005c00de 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -44,7 +44,7 @@ "minPtITSCut": "0.f", "etaITSCut": "1e10f", "minNITSClustersCut": "0", - "maxChi2PerClusterITS": "10000", + "maxChi2PerClusterITS": "100000", "minPtTPCCut": "0.1f", "etaTPCCut": "1.4f", "minNTPCClustersCut": "60", From 2da144cfe7342fc0a04dfe63ed67f2599a2ed886 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Tue, 9 Jan 2024 15:03:16 +0100 Subject: [PATCH 0301/1239] Add script to mimick containerized execution --- GRID/utils/runGRIDContainerized.sh | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 GRID/utils/runGRIDContainerized.sh diff --git a/GRID/utils/runGRIDContainerized.sh b/GRID/utils/runGRIDContainerized.sh new file mode 100755 index 000000000..6165a128f --- /dev/null +++ b/GRID/utils/runGRIDContainerized.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Runs a job containerized ... as would be the case on the GRID. +# Mimics what the AliEn job handler does. + +SCRIPT=$1 +[ $SCRIPT == "" ] && echo "Please provide a script to run" && exit 1 +echo "Trying to run script ${SCRIPT} in a container environment" + +# we just use the default singularity container +APPTAINER_CONTAINER=/cvmfs/alice.cern.ch/containers/fs/singularity/default + +# create workdir +WORK_DIR=$(mktemp -d /tmp/alien-job-XXXXXX) +echo "This job will be run in $WORK_DIR" + +# copy script to WORK_DIR +cp ${SCRIPT} ${WORK_DIR}/job.sh + +# export certificates (need to be created before) +ALIEN_CERTFILE=$(ls -t /tmp/tokencert_*.pem 2> /dev/null | head -n 1) +ALIEN_KEYFILE=$(ls -t /tmp/tokenkey_*.pem 2> /dev/null | head -n 1) + +[ "${ALIEN_CERTFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 +[ "${ALIEN_KEYFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 + +JALIEN_TOKEN_CERT=$(awk '! /^-/{print $0}' ${ALIEN_CERTFILE} | tr -d '\n') +JALIEN_TOKEN_KEY=$(awk '! /^-/{print $0}' ${ALIEN_KEYFILE} | tr -d '\n') + +echo "JALIEN_TOKEN_CERT=${JALIEN_TOKEN_CERT}" > ${WORK_DIR}/envfile +echo "JALIEN_TOKEN_KEY=${JALIEN_TOKEN_KEY}" >> ${WORK_DIR}/envfile + +# launch job = script inside the container in the workdir +/cvmfs/alice.cern.ch/containers/bin/apptainer/current/bin/apptainer exec -C -B /cvmfs:/cvmfs,${WORK_DIR}:/workdir \ + --pwd /workdir --env-file ${WORK_DIR}/envfile ${APPTAINER_CONTAINER} /workdir/job.sh From 25c3abc0fbc6c49c80eb015add3ebd9a6cec08fb Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 18 Dec 2023 13:55:56 +0100 Subject: [PATCH 0302/1239] Adjust mem estimates for sgnsim and tpcdigi Hoping to reduce zombie jobs and expired jobs for PbPb productions on GRID --- MC/bin/o2dpg_sim_workflow.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 72b0adb8f..e9c04e392 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -539,6 +539,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # translate here collision type to PDG COLTYPE=args.col + havePbPb = (COLTYPE == 'PbPb' or (doembedding and COLTYPEBKG == "PbPb")) if COLTYPE == 'pp': PDGA=2212 # proton @@ -668,7 +669,8 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): signalneeds = signalneeds + [ BKGtask['name'] ] else: signalneeds = signalneeds + [ BKG_HEADER_task['name'] ] - SGNtask=createTask(name='sgnsim_'+str(tf), needs=signalneeds, tf=tf, cwd='tf'+str(tf), lab=["GEANT"], relative_cpu=7/8, n_workers=NWORKERS, mem='2000') + sgnmem = 6000 if COLTYPE == 'PbPb' else 4000 + SGNtask=createTask(name='sgnsim_'+str(tf), needs=signalneeds, tf=tf, cwd='tf'+str(tf), lab=["GEANT"], relative_cpu=7/8, n_workers=NWORKERS, mem=str(sgnmem)) SGNtask['cmd']='${O2_ROOT}/bin/o2-sim -e ' + str(SIMENGINE) + ' ' + str(MODULES) + ' -n ' + str(NSIGEVENTS) + ' --seed ' + str(TFSEED) \ + ' --field ' + str(BFIELD) + ' -j ' + str(NWORKERS) + ' -g ' + str(GENERATOR) \ + ' ' + str(TRIGGER) + ' ' + str(CONFKEY) + ' ' + str(INIFILE) \ @@ -817,8 +819,9 @@ def putConfigValuesNew(listOfMainKeys=[], localCF = {}): if usebkgcache: tpcdigineeds += [ BKG_HITDOWNLOADER_TASKS['TPC']['name'] ] + tpcdigimem = 12000 if havePbPb else 9000 TPCDigitask=createTask(name='tpcdigi_'+str(tf), needs=tpcdigineeds, - tf=tf, cwd=timeframeworkdir, lab=["DIGI"], cpu=NWORKERS, mem='9000') + tf=tf, cwd=timeframeworkdir, lab=["DIGI"], cpu=NWORKERS, mem=str(tpcdigimem)) TPCDigitask['cmd'] = ('','ln -nfs ../bkg_HitsTPC.root . ;')[doembedding] TPCDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + ' --tpc-lanes ' + str(NWORKERS) \ @@ -961,7 +964,7 @@ def getDigiTaskName(det): + tpc_corr_scaling_options workflow['stages'].append(TPCRECOtask) - havePbPb = (COLTYPE == 'PbPb' or (doembedding and COLTYPEBKG == "PbPb")) + ITSMemEstimate = 12000 if havePbPb else 2000 # PbPb has much large mem requirement for now (in worst case) ITSRECOtask=createTask(name='itsreco_'+str(tf), needs=[getDigiTaskName("ITS")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem=str(ITSMemEstimate)) From b7bae82c3d21c4a1dffd3d54d543230eb56ad77f Mon Sep 17 00:00:00 2001 From: swenzel Date: Wed, 10 Jan 2024 16:15:01 +0100 Subject: [PATCH 0303/1239] Take field for sim from CCDB The field CCDB object is created as first step before simulation and Geant should just use this for better consistency. --- MC/bin/o2dpg_sim_workflow.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index e9c04e392..92a401a27 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -437,7 +437,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): BKGtask=createTask(name='bkgsim', lab=["GEANT"], needs=[BKG_CONFIG_task['name'], GRP_TASK['name']], cpu=NWORKERS ) BKGtask['cmd']='${O2_ROOT}/bin/o2-sim -e ' + SIMENGINE + ' -j ' + str(NWORKERS) + ' -n ' + str(NBKGEVENTS) \ + ' -g ' + str(GENBKG) + ' ' + str(MODULES) + ' -o bkg ' + str(INIBKG) \ - + ' --field ' + str(BFIELD) + ' ' + str(CONFKEYBKG) \ + + ' --field ccdb ' + str(CONFKEYBKG) \ + ('',' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run) \ + ' --vertexMode kCCDB' @@ -612,9 +612,8 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # ATTENTION: CHANGING THE PARAMETERS/CUTS HERE MIGHT INVALIDATE THE QED INTERACTION RATES USED ELSEWHERE # ######################################################################################################## - QED_task['cmd'] = 'o2-sim -e TGeant3 --field ' + str(BFIELD) + ' \ - -j ' + str('1') + ' -o qed_' + str(tf) + ' \ - -n ' + str(NEventsQED) + ' -m PIPE ITS MFT FT0 FV0 FDD ' \ + QED_task['cmd'] = 'o2-sim -e TGeant3 --field ccdb -j ' + str('1') + ' -o qed_' + str(tf) \ + + ' -n ' + str(NEventsQED) + ' -m PIPE ITS MFT FT0 FV0 FDD ' \ + ('', ' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run) \ + ' --seed ' + str(TFSEED) \ + ' -g extgen --configKeyValues \"GeneratorExternal.fileName=$O2_ROOT/share/Generators/external/QEDLoader.C;QEDGenParam.yMin=-7;QEDGenParam.yMax=7;QEDGenParam.ptMin=0.001;QEDGenParam.ptMax=1.;Diamond.width[2]=6.\"' # + (' ',' --fromCollContext collisioncontext.root')[args.pregenCollContext] @@ -672,7 +671,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): sgnmem = 6000 if COLTYPE == 'PbPb' else 4000 SGNtask=createTask(name='sgnsim_'+str(tf), needs=signalneeds, tf=tf, cwd='tf'+str(tf), lab=["GEANT"], relative_cpu=7/8, n_workers=NWORKERS, mem=str(sgnmem)) SGNtask['cmd']='${O2_ROOT}/bin/o2-sim -e ' + str(SIMENGINE) + ' ' + str(MODULES) + ' -n ' + str(NSIGEVENTS) + ' --seed ' + str(TFSEED) \ - + ' --field ' + str(BFIELD) + ' -j ' + str(NWORKERS) + ' -g ' + str(GENERATOR) \ + + ' --field ccdb -j ' + str(NWORKERS) + ' -g ' + str(GENERATOR) \ + ' ' + str(TRIGGER) + ' ' + str(CONFKEY) + ' ' + str(INIFILE) \ + ' -o ' + signalprefix + ' ' + embeddinto \ + ('', ' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run) \ From f9f636ffd3d4be74c42197f03ce280b5bcf20394 Mon Sep 17 00:00:00 2001 From: swenzel Date: Thu, 11 Jan 2024 11:22:33 +0100 Subject: [PATCH 0304/1239] Script determining number of produced MC events --- MC/bin/o2dpg_determine_eventstat.py | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 MC/bin/o2dpg_determine_eventstat.py diff --git a/MC/bin/o2dpg_determine_eventstat.py b/MC/bin/o2dpg_determine_eventstat.py new file mode 100755 index 000000000..08f3c81cc --- /dev/null +++ b/MC/bin/o2dpg_determine_eventstat.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# +# Script that determines final accounting numbers / event statistics +# for reporting back to MonaLisa. +# +# Analyses the AO2D / kinematics output of an O2DPG simulation run +# and creates a file of the form +# +# inputN_passedN_errorsN_outputN.stat +# +# which is picked up and used by the MonaLisa system. +# +# See discussion in https://alice.its.cern.ch/jira/browse/O2-4553; +# Here outputN would be the number of events/collisions produced in this job. + +import ROOT +import argparse +import os +import re + +parser = argparse.ArgumentParser(description='', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + +parser.add_argument('-f','--aod-file', default="AO2D.root", help='AO2D file to check') +args = parser.parse_args() + +def write_stat_file(eventcount): + """ + writes a file conforming to MonaLisa convention + """ + + filename = '0_0_0_' + str(eventcount) + '.stat' + # touche/create a new file + with open(filename, 'w') as f: + print ("#This file is autogenerated", file=f) + print ("#It tells MonaLisa about the number of produced MC events", file=f) + print ("#Numer of MC collisions in AOD : " + str(eventcount), file=f) + +def read_collisioncontext_eventcount(file): + """ + determines MC eventcount from collision context files + """ + pass + +def find_files_matching_pattern(directory='.', pattern='.*'): + matching_files = [] + + # Walk through the directory and its subdirectories + for root, dirs, files in os.walk(directory): + for file_name in files: + # Check if the filename matches the regular expression pattern + if re.match(pattern, file_name): + matching_files.append(os.path.join(root, file_name)) + + return matching_files + +def read_GEANT_eventcount(file): + # Open the ROOT file + eventcount = 0 + tfile = ROOT.TFile.Open(file) + if tfile: + simtree = tfile.Get("o2sim") + if simtree and isinstance(simtree, ROOT.TTree): + eventcount = simtree.GetEntries() + + tfile.Close() + return eventcount + +def read_accumulated_GEANT_eventcount(directory = "."): + """ + Determines the MC eventcount from GEANT kinematics files sitting + in directory/tfX/ subdirectories. + """ + pattern_to_match = r'sgn.*_Kine.root' + kine_files = find_files_matching_pattern(directory, pattern_to_match) + eventcount = 0 + for f in kine_files: + eventcount = eventcount + read_GEANT_eventcount(f) + return eventcount + +def read_AO2D_eventcount(file): + """ + determines MC eventcount from (final) AO2D file + """ + eventcount = 0 + + # Open the ROOT file + tfile = ROOT.TFile.Open(file) + + # Get the list of keys (TKeys) in the ROOT files + keys = tfile.GetListOfKeys() + + # Iterate through the keys "DF_" keys and accumulate + # stored MC collisions + for key in keys: + key_name = key.GetName() + if key_name.startswith("DF_"): + obj = key.ReadObj() + # the O2mccollision tree contains the simulated collisions + coltree = obj.Get("O2mccollision") + if coltree and isinstance(coltree, ROOT.TTree): + eventcount = eventcount + coltree.GetEntries() + + # Close the files + tfile.Close() + return eventcount + +AO2D_eventcount = read_AO2D_eventcount(args.aod_file) +GEANT_eventcount = read_accumulated_GEANT_eventcount() +if AO2D_eventcount != GEANT_eventcount: + print ("WARN: AO2D MC event count and GEANT event count differ") + +write_stat_file(AO2D_eventcount) \ No newline at end of file From 185db664556e0b9f3a9b9c514340b6ee2e9db2db Mon Sep 17 00:00:00 2001 From: swenzel Date: Thu, 11 Jan 2024 11:27:48 +0100 Subject: [PATCH 0305/1239] Produce MonaLisa stat file in O2DPG workflows produces the MonaLisa event accounting file during the final AO2D step --- MC/bin/o2dpg_sim_workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 92a401a27..e2eb391cf 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1441,9 +1441,11 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # AOD merging as one global final step aodmergerneeds = ['aod_' + str(tf) for tf in range(1, NTIMEFRAMES + 1)] AOD_merge_task = createTask(name='aodmerge', needs = aodmergerneeds, lab=["AOD"], mem='2000', cpu='1') -AOD_merge_task['cmd'] = ' [ -f aodmerge_input.txt ] && rm aodmerge_input.txt; ' +AOD_merge_task['cmd'] = ' set -e ; [ -f aodmerge_input.txt ] && rm aodmerge_input.txt; ' AOD_merge_task['cmd'] += ' for i in `seq 1 ' + str(NTIMEFRAMES) + '`; do echo "tf${i}/AO2D.root" >> aodmerge_input.txt; done; ' AOD_merge_task['cmd'] += ' o2-aod-merger --input aodmerge_input.txt --output AO2D.root' +# produce MonaLisa event stat file +AOD_merge_task['cmd'] += ' ; ${O2DPG_ROOT}/MC/bin/o2dpg_determine_eventstat.py' workflow['stages'].append(AOD_merge_task) job_merging = False From ef0e84d4d13fcafb86b96895ca370894025619fa Mon Sep 17 00:00:00 2001 From: Benedikt Volkel Date: Thu, 26 Oct 2023 12:31:52 +0200 Subject: [PATCH 0306/1239] [WFRunner] Estimate resources dynamically * introduce functionality to update resource estimates based on values which are collected during monitoring (the same that go into pipeline_metric_.log) * disentangle resource management from WorkflowExecutor * manage resources via ResourceManager object * memory and cpu * semaphores * number of parallel tasks to be launched * niceness * main interfaces of ResourceManager * ok_to_submit * book * unbook * add_monitored_resources * group corresponding tasks together to be able to update resource estimates for upcoming tasks * TaskResources as container of a task's resource estimate holding * resource estimates (initial and new estimates) * references to Semaphore object (optional * reference to list of related tasks (optional) * make sure, final niceness assigned to a launched process is the same that the TaskResources hold for that process * enable functionality with --dynamic-resources * add more comments to classes and functions * update --cgroup option to require entire path --> handle both cgroups v1 and cgroups v2 --- MC/bin/o2_dpg_workflow_runner.py | 772 ++++++++++++++++++++++--------- 1 file changed, 541 insertions(+), 231 deletions(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index 766cfb9e4..cc4ceb57d 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -4,7 +4,6 @@ import re import subprocess -import shlex import time import json import logging @@ -33,7 +32,7 @@ from o2dpg_workflow_utils import read_workflow # defining command line options -parser = argparse.ArgumentParser(description='Parallel execution of a (O2-DPG) DAG data/job pipeline under resource contraints.', +parser = argparse.ArgumentParser(description='Parallel execution of a (O2-DPG) DAG data/job pipeline under resource contraints.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-f','--workflowfile', help='Input workflow file name', required=True) @@ -47,18 +46,26 @@ parser.add_argument('--produce-script', help='Produces a shell script that runs the workflow in serialized manner and quits.') parser.add_argument('--rerun-from', help='Reruns the workflow starting from given task (or pattern). All dependent jobs will be rerun.') parser.add_argument('--list-tasks', help='Simply list all tasks by name and quit.', action='store_true') -parser.add_argument('--update-resources', dest="update_resources", help='Read resource estimates from a JSON and apply where possible.') -parser.add_argument('--mem-limit', help='Set memory limit as scheduling constraint (in MB)', default=0.9*max_system_mem/1024./1024) -parser.add_argument('--cpu-limit', help='Set CPU limit (core count)', default=8) +# Resources +parser.add_argument('--update-resources', dest="update_resources", help='Read resource estimates from a JSON and apply where possible.') +parser.add_argument("--dynamic-resources", dest="dynamic_resources", action="store_true", help="Update reources estimates of task based on finished related tasks") # derive resources dynamically +parser.add_argument('--optimistic-resources', dest="optimistic_resources", action="store_true", help="Try to run workflow even though resource limits might underestimate resource needs of some tasks") +parser.add_argument("--n-backfill", dest="n_backfill", type=int, default=1) +parser.add_argument('--mem-limit', help='Set memory limit as scheduling constraint (in MB)', default=0.9*max_system_mem/1024./1024, type=float) +parser.add_argument('--cpu-limit', help='Set CPU limit (core count)', default=8, type=float) parser.add_argument('--cgroup', help='Execute pipeline under a given cgroup (e.g., 8coregrid) emulating resource constraints. This m\ ust exist and the tasks file must be writable to with the current user.') + +# run control, webhooks parser.add_argument('--stdout-on-failure', action='store_true', help='Print log files of failing tasks to stdout,') parser.add_argument('--webhook', help=argparse.SUPPRESS) # log some infos to this webhook channel parser.add_argument('--checkpoint-on-failure', help=argparse.SUPPRESS) # debug option making a debug-tarball and sending to specified address # argument is alien-path parser.add_argument('--retry-on-failure', help=argparse.SUPPRESS, default=0) # number of times a failing task is retried parser.add_argument('--no-rootinit-speedup', help=argparse.SUPPRESS, action='store_true') # disable init of ROOT environment vars to speedup init/startup + +# Logging parser.add_argument('--action-logfile', help='Logfilename for action logs. If none given, pipeline_action_#PID.log will be used') parser.add_argument('--metric-logfile', help='Logfilename for metric logs. If none given, pipeline_metric_#PID.log will be used') parser.add_argument('--production-mode', action='store_true', help='Production mode') @@ -68,7 +75,7 @@ def setup_logger(name, log_file, level=logging.INFO): """To setup as many loggers as you want""" - handler = logging.FileHandler(log_file, mode='w') + handler = logging.FileHandler(log_file, mode='w') handler.setFormatter(formatter) logger = logging.getLogger(name) @@ -130,10 +137,10 @@ def getChildProcs(basepid): proc=psutil.Process(int(p)) except psutil.NoSuchProcess: continue - + plist.append(proc) return plist - + # # Code section to find all topological orderings # of a DAG. This is used to know when we can schedule @@ -142,23 +149,23 @@ def getChildProcs(basepid): # class to represent a graph object class Graph: - + # Constructor def __init__(self, edges, N): - + # A List of Lists to represent an adjacency list self.adjList = [[] for _ in range(N)] - + # stores in-degree of a vertex # initialize in-degree of each vertex by 0 self.indegree = [0] * N - + # add edges to the undirected graph for (src, dest) in edges: - + # add an edge from source to destination self.adjList[src].append(dest) - + # increment in-degree of destination vertex by 1 self.indegree[dest] = self.indegree[dest] + 1 @@ -166,47 +173,47 @@ def __init__(self, edges, N): def findAllTopologicalOrders(graph, path, discovered, N, allpaths, maxnumber=1): if len(allpaths) >= maxnumber: return - + # do for every vertex for v in range(N): - + # proceed only if in-degree of current node is 0 and # current node is not processed yet if graph.indegree[v] == 0 and not discovered[v]: - + # for every adjacent vertex u of v, reduce in-degree of u by 1 for u in graph.adjList[v]: graph.indegree[u] = graph.indegree[u] - 1 - + # include current node in the path and mark it as discovered path.append(v) discovered[v] = True - + # recur findAllTopologicalOrders(graph, path, discovered, N, allpaths) - + # backtrack: reset in-degree information for the current node for u in graph.adjList[v]: graph.indegree[u] = graph.indegree[u] + 1 - + # backtrack: remove current node from the path and # mark it as undiscovered path.pop() discovered[v] = False - + # record valid ordering if len(path) == N: allpaths.append(path.copy()) - - + + # get all topological orderings of a given DAG as a list def printAllTopologicalOrders(graph, maxnumber=1): # get number of nodes in the graph N = len(graph.adjList) - + # create an auxiliary space to keep track of whether vertex is discovered discovered = [False] * N - + # list to store the topological order path = [] allpaths = [] @@ -218,21 +225,23 @@ def printAllTopologicalOrders(graph, maxnumber=1): # find all tasks that depend on a given task (id); when a cache # dict is given we can fill for the whole graph in one pass... -def find_all_dependent_tasks(possiblenexttask, tid, cache={}): - c=cache.get(tid) +def find_all_dependent_tasks(possiblenexttask, tid, cache=None): + c=cache.get(tid) if cache else None if c!=None: return c - + daughterlist=[tid] # possibly recurse for n in possiblenexttask[tid]: - c = cache.get(n) + c = cache.get(n) if cache else None if c == None: c = find_all_dependent_tasks(possiblenexttask, n, cache) daughterlist = daughterlist + c - cache[n]=c + if cache is not None: + cache[n]=c - cache[tid]=daughterlist + if cache is not None: + cache[tid]=daughterlist return list(set(daughterlist)) @@ -250,12 +259,12 @@ def analyseGraph(edges, nodes): nextjobtrivial[e[0]].append(e[1]) if nextjobtrivial[-1].count(e[1]): nextjobtrivial[-1].remove(e[1]) - + # find topological orderings of the graph # create a graph from edges graph = Graph(edges, N) orderings = printAllTopologicalOrders(graph) - + return (orderings, nextjobtrivial) @@ -282,7 +291,7 @@ def draw_workflow(workflowspec): dot.edge(str(fromindex), str(toindex)) dot.render('workflow.gv') - + # builds the graph given a "taskuniverse" list # builds accompagnying structures tasktoid and idtotask def build_graph(taskuniverse, workflowspec): @@ -295,9 +304,9 @@ def build_graph(taskuniverse, workflowspec): nodes.append(tasktoid[t[0]['name']]) for n in t[0]['needs']: edges.append((tasktoid[n], tasktoid[t[0]['name']])) - + return (edges, nodes) - + # loads json into dict, e.g. for workflow specification def load_json(workflowfile): @@ -333,7 +342,7 @@ def task_matches_labels(t): if targetlabels.count(l)!=0: return True return False - + # The following sequence of operations works and is somewhat structured. # However, it builds lookups used elsewhere as well, so some CPU might be saved by reusing # some structures across functions or by doing less passes on the data. @@ -341,7 +350,7 @@ def task_matches_labels(t): # helper lookup tasknametoid = { t['name']:i for i, t in enumerate(workflowspec['stages'],0) } - # check if a task can be run at all + # check if a task can be run at all # or not due to missing requirements def canBeDone(t,cache={}): ok = True @@ -393,17 +402,17 @@ def needed_by_targets(name): return transformedworkflowspec -# builds topological orderings (for each timeframe) +# builds topological orderings (for each timeframe) def build_dag_properties(workflowspec): globaltaskuniverse = [ (l, i) for i, l in enumerate(workflowspec['stages'], 1) ] timeframeset = set( l['timeframe'] for l in workflowspec['stages'] ) edges, nodes = build_graph(globaltaskuniverse, workflowspec) tup = analyseGraph(edges, nodes.copy()) - # + # global_next_tasks = tup[1] - + dependency_cache = {} # weight influences scheduling order can be anything user defined ... for the moment we just prefer to stay within a timeframe # then take the number of tasks that depend on a task as further weight @@ -411,7 +420,7 @@ def build_dag_properties(workflowspec): # TODO: make this a policy of the runner to study different strategies def getweight(tid): return (globaltaskuniverse[tid][0]['timeframe'], len(find_all_dependent_tasks(global_next_tasks, tid, dependency_cache))) - + task_weights = [ getweight(tid) for tid in range(len(globaltaskuniverse)) ] for tid in range(len(globaltaskuniverse)): @@ -420,6 +429,7 @@ def getweight(tid): # print (global_next_tasks) return { 'nexttasks' : global_next_tasks, 'weights' : task_weights, 'topological_ordering' : tup[0] } + # update the resource estimates of a workflow based on resources given via JSON def update_resource_estimates(workflow, resource_json): resource_dict = load_json(resource_json) @@ -427,7 +437,6 @@ def update_resource_estimates(workflow, resource_json): for task in stages: if task["timeframe"] >= 1: - tf = task["timeframe"] name = "_".join(task["name"].split("_")[:-1]) else: name = task["name"] @@ -497,6 +506,328 @@ def get_alienv_software_environment(packagestring): # # functions for execution; encapsulated in a WorkflowExecutor class # + +class Semaphore: + """ + Object that can be used as semaphore + """ + def __init__(self): + self.locked = False + def lock(self): + self.locked = True + def unlock(self): + self.locked = False + + +class ResourceBoundaries: + """ + Container holding global resource properties + """ + def __init__(self, cpu_limit, mem_limit, dynamic_resources=False, optimistic_resources=False): + self.cpu_limit = cpu_limit + self.mem_limit = mem_limit + self.dynamic_resources = dynamic_resources + # if this is set, tasks that would normally go beyond the resource limits will tried to be run in any case + self.optimistic_resource = optimistic_resources + + +class TaskResources: + """ + Container holding resources of a single task + """ + def __init__(self, tid, name, cpu, mem, resource_boundaries): + # the task ID belonging to these resources + self.tid = tid + self.name = name + # original CPUs/MEM assigned (persistent) + self.cpu_assigned_original = cpu + self.mem_assigned_original = mem + # CPUs/MEM assigned (transient) + self.cpu_assigned = cpu + self.mem_assigned = mem + # global resource settings + self.resource_boundaries = resource_boundaries + # sampled resources of this + self.cpu_sampled = None + self.mem_sampled = None + # Set these after a task has finished to compute new estomates for related tasks + self.walltime = None + self.cpu_taken = None + self.mem_taken = None + # collected during monitoring + self.time_collect = [] + self.cpu_collect = [] + self.mem_collect = [] + # linked to other resources of task that are of the same type as this one + self.related_tasks = None + # can assign a semaphore + self.semaphore = None + # the task's nice value + self.nice_value = None + # whether or not the task's resources are currently booked + self.booked = False + + @property + def is_done(self): + return self.time_collect and not self.booked + + def add(self, time_passed, cpu, mem): + """ + Brief interface to add resources that were measured after time_passed + """ + self.time_collect.append(time_passed) + self.cpu_collect.append(cpu) + self.mem_collect.append(mem) + + def sample_resources(self): + """ + If this task is done, sample CPU and MEM for all related tasks that have not started yet + """ + if not self.is_done: + return + + if len(self.time_collect) < 3: + # Consider at least 3 points to sample from + self.cpu_sampled = self.cpu_assigned + self.mem_sampled = self.mem_assigned + actionlogger.debug("Task %s has not enough points (< 3) to sample resources, setting to previosuly assigned values.", self.name) + else: + # take the time deltas and leave out the very first CPU measurent which is not meaningful, + # at least when it domes from psutil.Proc.cpu_percent(interval=None) + time_deltas = [self.time_collect[i+1] - self.time_collect[i] for i in range(len(self.time_collect) - 1)] + cpu = sum([cpu * time_delta for cpu, time_delta in zip(self.cpu_collect[1:], time_deltas) if cpu >= 0]) + self.cpu_sampled = cpu / sum(time_deltas) + self.mem_sampled = max(self.mem_collect) + + mem_sampled = 0 + cpu_sampled = [] + for res in self.related_tasks: + if res.is_done: + mem_sampled = max(mem_sampled, res.mem_sampled) + cpu_sampled.append(res.cpu_sampled) + cpu_sampled = sum(cpu_sampled) / len(cpu_sampled) + + # This task ran already with the assigned resources, so let's set it to the limit + if cpu_sampled > self.resource_boundaries.cpu_limit: + actionlogger.warning("Sampled CPU (%.2f) exceeds assigned CPU limit (%.2f)", cpu_sampled, self.resource_boundaries.cpu_limit) + cpu_sampled = self.resource_boundaries.cpu_limit + if mem_sampled > self.resource_boundaries.mem_limit: + actionlogger.warning("Sampled MEM (%.2f) exceeds assigned MEM limit (%.2f)", mem_sampled, self.resource_boundaries.mem_limit) + mem_sampled = self.resource_boundaries.mem_limit + + if mem_sampled <= 0: + actionlogger.debug("Sampled memory for %s is %.2f <= 0, setting to previously assigned value %.2f", self.name, mem_sampled, self.mem_assigned) + mem_sampled = self.mem_assigned + if cpu_sampled < 0: + actionlogger.debug("Sampled CPU for %s is %.2f < 0, setting to previously assigned value %.2f", self.name, cpu_sampled, self.cpu_assigned) + cpu_sampled = self.cpu_assigned + for res in self.related_tasks: + if res.is_done or res.booked: + continue + res.cpu_assigned = cpu_sampled + res.mem_assigned = mem_sampled + + +class ResourceManager: + """ + Central class to manage resources + + - CPU limits + - MEM limits + - Semaphores + + Entrypoint to set and to query for resources to be updated. + + Can be asked whether a certain task can be run under current resource usage. + Book and unbook resources. + """ + def __init__(self, cpu_limit, mem_limit, procs_parallel_max=100, dynamic_resources=False, optimistic_resources=False): + """ + Initialise members with defaults + """ + # hold TaskResources of all tasks + self.resources = [] + + # helper dictionaries holding common objects which will be distributed to single TaskResources objects + # to avoid further lookup and at the same time to share the same common objects + self.resources_related_tasks_dict = {} + self.semaphore_dict = {} + + # one common object that holds global resource settings such as CPU and MEM limits + self.resource_boundaries = ResourceBoundaries(cpu_limit, mem_limit, dynamic_resources, optimistic_resources) + + # register resources that are booked under default nice value + self.cpu_booked = 0 + self.mem_booked = 0 + # number of tasks currently booked + self.n_procs = 0 + + # register resources that are booked under high nice value + self.cpu_booked_backfill = 0 + self.mem_booked_backfill = 0 + # number of tasks currently booked under high nice value + self.n_procs_backfill = 0 + + # the maximum number of tasks that run at the same time + self.procs_parallel_max = procs_parallel_max + + # get the default nice value of this python script + self.nice_default = os.nice(0) + # add 19 to get nice value of low-priority tasks + self.nice_backfill = self.nice_default + 19 + + def add_task_resources(self, name, related_tasks_name, cpu, mem, semaphore_string=None): + """ + Construct and Add a new TaskResources object + """ + resources = TaskResources(len(self.resources), name, cpu, mem, self.resource_boundaries) + if cpu > self.resource_boundaries.cpu_limit or mem > self.resource_boundaries.mem_limit: + actionlogger.warning(f"Resource estimates of id {len(self.resources)} overestimates limits, CPU limit: {self.resource_boundaries.cpu_limit}, MEM limit: {self.resource_boundaries.mem_limit}; might not run") + if not self.resource_boundaries.optimistic_resources: + # exit if we don't dare to try + exit(1) + # or we do dare, let's see what happens... + actionlogger.info("We will try to run this task anyway with maximum available resources") + + self.resources.append(resources) + # do the following to have the same Semaphore object for all corresponding TaskResources so that we do not need a lookup + if semaphore_string: + if semaphore_string not in self.semaphore_dict: + self.semaphore_dict[semaphore_string] = Semaphore() + resources.semaphore = self.semaphore_dict[semaphore_string] + + # do the following to give each TaskResources a list of the related tasks so we do not need an additional lookup + if related_tasks_name: + if related_tasks_name not in self.resources_related_tasks_dict: + # assigned list is [valid top be used, list of CPU, list of MEM, list of walltimes of each related task, list of processes that ran in parallel on average, list of taken CPUs, list of assigned CPUs, list of tasks finished in the meantime] + self.resources_related_tasks_dict[related_tasks_name] = [] + self.resources_related_tasks_dict[related_tasks_name].append(resources) + resources.related_tasks = self.resources_related_tasks_dict[related_tasks_name] + + def add_monitored_resources(self, tid, time_delta_since_start, cpu, mem): + self.resources[tid].add(time_delta_since_start, cpu, mem) + + def book(self, tid, nice_value): + """ + Book the resources of this task with given nice value + + The final nice value is determined by the final submission and could be different. + This can happen if the nice value should have been changed while that is not allowed by the system. + """ + res = self.resources[tid] + # take the nice value that was previously assigned when resources where checked last time + previous_nice_value = res.nice_value + + if previous_nice_value is None: + # this has not been checked ever if it was ok to be submitted + actionlogger.warning("Task ID %d has never been checked for resources. Treating as backfill", tid) + nice_value = self.nice_backfill + elif res.nice_value != nice_value: + actionlogger.warning("Task ID %d has was last time checked for a different nice value (%d) but is now submitted with (%d).", tid, res.nice_value, nice_value) + + res.nice_value = nice_value + res.booked = True + if res.semaphore is not None: + res.semaphore.lock() + if nice_value != self.nice_default: + self.n_procs_backfill += 1 + self.cpu_booked_backfill += res.cpu_assigned + self.mem_booked_backfill += res.mem_assigned + return + self.n_procs += 1 + self.cpu_booked += res.cpu_assigned + self.mem_booked += res.mem_assigned + + def unbook(self, tid): + """ + Unbook the reources of this task + """ + res = self.resources[tid] + res.booked = False + if self.resource_boundaries.dynamic_resources: + res.sample_resources() + if res.semaphore is not None: + res.semaphore.unlock() + if res.nice_value != self.nice_default: + self.cpu_booked_backfill -= res.cpu_assigned + self.mem_booked_backfill -= res.mem_assigned + self.n_procs_backfill -= 1 + if self.n_procs_backfill <= 0: + self.cpu_booked_backfill = 0 + self.mem_booked_backfill = 0 + return + self.n_procs -= 1 + self.cpu_booked -= res.cpu_assigned + self.mem_booked -= res.mem_assigned + if self.n_procs <= 0: + self.cpu_booked = 0 + self.mem_booked = 0 + + def ok_to_submit(self, tids): + """ + This generator yields the tid and nice value tuple from the list of task ids that should be checked + """ + tids_copy = tids.copy() + + def ok_to_submit_default(res): + """ + Return default nice value if conditions are met, None otherwise + """ + # analyse CPU + okcpu = (self.cpu_booked + res.cpu_assigned <= self.resource_boundaries.cpu_limit) + # analyse MEM + okmem = (self.mem_booked + res.mem_assigned <= self.resource_boundaries.mem_limit) + actionlogger.debug ('Condition check --normal-- for ' + str(res.tid) + ':' + res.name + ' CPU ' + str(okcpu) + ' MEM ' + str(okmem)) + return self.nice_default if (okcpu and okmem) else None + + def ok_to_submit_backfill(res, backfill_cpu_factor=1.5, backfill_mem_factor=1.5): + """ + Return backfill nice value if conditions are met, None otherwise + """ + if self.n_procs_backfill >= args.n_backfill: + return None + + if res.cpu_assigned > 0.9 * self.resource_boundaries.cpu_limit or res.mem_assigned / self.resource_boundaries.cpu_limit >= 1900: + return None + + # analyse CPU + okcpu = (self.cpu_booked_backfill + res.cpu_assigned <= self.resource_boundaries.cpu_limit) + okcpu = okcpu and (self.cpu_booked + self.cpu_booked_backfill + res.cpu_assigned <= backfill_cpu_factor * self.resource_boundaries.cpu_limit) + # analyse MEM + okmem = (self.mem_booked + self.mem_booked_backfill + res.mem_assigned <= backfill_mem_factor * self.resource_boundaries.mem_limit) + actionlogger.debug ('Condition check --backfill-- for ' + str(res.tid) + ':' + res.name + ' CPU ' + str(okcpu) + ' MEM ' + str(okmem)) + + return self.nice_backfill if (okcpu and okmem) else None + + if self.n_procs + self.n_procs_backfill >= self.procs_parallel_max: + # in this case, nothing can be done + return + + for ok_to_submit_impl, should_break in ((ok_to_submit_default, True), (ok_to_submit_backfill, False)): + tid_index = 0 + while tid_index < len(tids_copy): + + tid = tids_copy[tid_index] + res = self.resources[tid] + + actionlogger.info("Setup resources for task %s, cpu: %f, mem: %f", res.name, res.cpu_assigned, res.mem_assigned) + tid_index += 1 + + if (res.semaphore is not None and res.semaphore.locked) or res.booked: + continue + + nice_value = ok_to_submit_impl(res) + if nice_value is not None: + # if we get a non-None nice value, it means that this task is good to go + res.nice_value = nice_value + # yield the tid and its assigned nice value + yield tid, nice_value + + elif should_break: + # break here if resources of the next task do not fit + break + + class WorkflowExecutor: # Constructor def __init__(self, workflowfile, args, jmax=100): @@ -510,6 +841,7 @@ def __init__(self, workflowfile, args, jmax=100): actionlogger.info("Applying global environment from init section " + str(e) + " : " + str(self.globalenv[e])) os.environ[e] = str(self.globalenv[e]) + # only keep those tasks that are necessary to be executed based on user's filters self.workflowspec = filter_workflow(self.workflowspec, args.target_tasks, args.target_labels) if not self.workflowspec['stages']: @@ -518,7 +850,8 @@ def __init__(self, workflowfile, args, jmax=100): exit (0) print ('Workflow is empty. Nothing to do') exit (0) - + + # construct the DAG, compute task weights workflow = build_dag_properties(self.workflowspec) if args.visualize_workflow: draw_workflow(self.workflowspec) @@ -526,29 +859,29 @@ def __init__(self, workflowfile, args, jmax=100): self.taskweights = workflow['weights'] self.topological_orderings = workflow['topological_ordering'] self.taskuniverse = [ l['name'] for l in self.workflowspec['stages'] ] - self.idtotask = [ 0 for l in self.taskuniverse ] + # construct task ID <-> task name lookup + self.idtotask = [ 0 for _ in self.taskuniverse ] self.tasktoid = {} - for i in range(len(self.taskuniverse)): - self.tasktoid[self.taskuniverse[i]]=i - self.idtotask[i]=self.taskuniverse[i] + for i, name in enumerate(self.taskuniverse): + self.tasktoid[name]=i + self.idtotask[i]=name if args.update_resources: update_resource_estimates(self.workflowspec, args.update_resources) - self.maxmemperid = [ self.workflowspec['stages'][tid]['resources']['mem'] for tid in range(len(self.taskuniverse)) ] - self.cpuperid = [ self.workflowspec['stages'][tid]['resources']['cpu'] for tid in range(len(self.taskuniverse)) ] - self.curmembooked = 0 - self.curcpubooked = 0 - self.curmembooked_backfill = 0 - self.curcpubooked_backfill = 0 - self.memlimit = float(args.mem_limit) # some configurable number - self.cpulimit = float(args.cpu_limit) + # construct the object that is in charge of resource management... + self.resource_manager = ResourceManager(args.cpu_limit, args.mem_limit, args.maxjobs, args.dynamic_resources, args.optimistic_resources) + for task in self.workflowspec['stages']: + # ...and add all initial resource estimates + global_task_name = self.get_global_task_name(task["name"]) + self.resource_manager.add_task_resources(task["name"], global_task_name, float(task["resources"]["cpu"]), float(task["resources"]["mem"]), task.get("semaphore")) + self.procstatus = { tid:'ToDo' for tid in range(len(self.workflowspec['stages'])) } self.taskneeds= { t:set(self.getallrequirements(t)) for t in self.taskuniverse } - self.stoponfailure = not (args.keep_going == True) + self.stoponfailure = not args.keep_going print ("Stop on failure ",self.stoponfailure) - self.max_jobs_parallel = int(jmax) - self.scheduling_iteration = 0 + + self.scheduling_iteration = 0 # count how often it was tried to schedule new tasks self.process_list = [] # list of currently scheduled tasks with normal priority self.backfill_process_list = [] # list of curently scheduled tasks with low backfill priority (not sure this is needed) self.pid_to_psutilsproc = {} # cache of putilsproc for resource monitoring @@ -556,20 +889,20 @@ def __init__(self, workflowfile, args, jmax=100): self.pid_to_connections = {} # we can auto-detect what connections are opened by which task (at least to some extent) signal.signal(signal.SIGINT, self.SIGHandler) signal.siginterrupt(signal.SIGINT, False) - self.nicevalues = [ os.nice(0) for tid in range(len(self.taskuniverse)) ] self.internalmonitorcounter = 0 # internal use self.internalmonitorid = 0 # internal use self.tids_marked_toretry = [] # sometimes we might want to retry a failed task (simply because it was "unlucky") and we put them here self.retry_counter = [ 0 for tid in range(len(self.taskuniverse)) ] # we keep track of many times retried already self.task_retries = [ self.workflowspec['stages'][tid].get('retry_count',0) for tid in range(len(self.taskuniverse)) ] # the per task specific "retry" number -> needs to be parsed from the JSON - self.semaphore_values = { self.workflowspec['stages'][tid].get('semaphore'):0 for tid in range(len(self.taskuniverse)) if self.workflowspec['stages'][tid].get('semaphore')!=None } # keeps current count of semaphores (defined in the json workflow). used to achieve user-defined "critical sections". self.alternative_envs = {} # mapping of taskid to alternative software envs (to be applied on a per-task level) # init alternative software environments self.init_alternative_software_environments() def SIGHandler(self, signum, frame): - # basically forcing shut down of all child processes + """ + basically forcing shut down of all child processes + """ actionlogger.info("Signal " + str(signum) + " caught") try: procs = psutil.Process().children(recursive=True) @@ -584,8 +917,8 @@ def SIGHandler(self, signum, frame): p.terminate() except (psutil.NoSuchProcess, psutil.AccessDenied): pass - - gone, alive = psutil.wait_procs(procs, timeout=3) + + _, alive = psutil.wait_procs(procs, timeout=3) for p in alive: try: actionlogger.info("Killing " + str(p)) @@ -595,38 +928,74 @@ def SIGHandler(self, signum, frame): exit (1) - def extract_global_environment(self, workflowspec): - """Checks if the workflow contains a dedicated init task - defining a global environment. Extract information and remove from workflowspec. + """ + Checks if the workflow contains a dedicated init task + defining a global environment. Extract information and remove from workflowspec. """ init_index = 0 # this has to be the first task in the workflow globalenv = {} if workflowspec['stages'][init_index]['name'] == '__global_init_task__': env = workflowspec['stages'][init_index].get('env', None) if env != None: - globalenv = { e : env[e] for e in env } + globalenv = { e : env[e] for e in env } del workflowspec['stages'][init_index] return globalenv - def getallrequirements(self, t): + def get_global_task_name(self, name): + """ + Get the global task name + + Tasks are related if only the suffix _ is different + """ + tokens = name.split("_") + try: + int(tokens[-1]) + return "_".join(tokens[:-1]) + except ValueError: + pass + return name + + def getallrequirements(self, task_name): + """ + get all requirement of a task by its name + """ l=[] - for r in self.workflowspec['stages'][self.tasktoid[t]]['needs']: - l.append(r) - l=l+self.getallrequirements(r) + for required_task_name in self.workflowspec['stages'][self.tasktoid[task_name]]['needs']: + l.append(required_task_name) + l=l+self.getallrequirements(required_task_name) return l - def get_done_filename(self, tid): + def get_logfile(self, tid): + """ + O2 taskwrapper logs task stdout and stderr to logfile .log + Get its exact path based on task ID + """ + # determines the logfile name for this task name = self.workflowspec['stages'][tid]['name'] workdir = self.workflowspec['stages'][tid]['cwd'] - # name and workdir define the "done" file as used by taskwrapper - # this assumes that taskwrapper is used to actually check if something is to be rerun - done_filename = workdir + '/' + name + '.log_done' - return done_filename + return os.path.join(workdir, f"{name}.log") + + def get_done_filename(self, tid): + """ + O2 taskwrapper leaves .log_done after a task has successfully finished + Get its exact path based on task ID + """ + return f"{self.get_logfile(tid)}_done" + + def get_resources_filename(self, tid): + """ + O2 taskwrapper leaves .log_time after a task is done + Get its exact path based on task ID + """ + return f"{self.get_logfile(tid)}_time" # removes the done flag from tasks that need to be run again def remove_done_flag(self, listoftaskids): + """ + Remove .log_done files to given task IDs + """ for tid in listoftaskids: done_filename = self.get_done_filename(tid) name=self.workflowspec['stages'][tid]['name'] @@ -636,13 +1005,23 @@ def remove_done_flag(self, listoftaskids): print ("Marking task " + name + " as to be done again") if os.path.exists(done_filename) and os.path.isfile(done_filename): os.remove(done_filename) - + # submits a task as subprocess and records Popen instance - def submit(self, tid, nice=os.nice(0)): + def submit(self, tid, nice): + """ + Submit a task + + 1. if needed, construct working directory if it does not yet exist + 2. update lookup structures flagging the task as being run + 3. set specific environment if requested for task + 4. construct psutil.Process from command line + 4.1 adjust the niceness of that process if requested + 5. return psutil.Process object + """ actionlogger.debug("Submitting task " + str(self.idtotask[tid]) + " with nice value " + str(nice)) c = self.workflowspec['stages'][tid]['cmd'] workdir = self.workflowspec['stages'][tid]['cwd'] - if not workdir=='': + if workdir: if os.path.exists(workdir) and not os.path.isdir(workdir): actionlogger.error('Cannot create working dir ... some other resource exists already') return None @@ -672,137 +1051,52 @@ def submit(self, tid, nice=os.nice(0)): p = psutil.Popen(['/bin/bash','-c',c], cwd=workdir, env=taskenv) try: p.nice(nice) - self.nicevalues[tid]=nice except (psutil.NoSuchProcess, psutil.AccessDenied): actionlogger.error('Couldn\'t set nice value of ' + str(p.pid) + ' to ' + str(nice)) - self.nicevalues[tid]=os.nice(0) - return p - - def ok_to_submit(self, tid, backfill=False): - softcpufactor=1 - softmemfactor=1 - if backfill: - softcpufactor=1.5 - sotmemfactor=1.5 - - # check semaphore - sem = self.workflowspec['stages'][tid].get('semaphore') - if sem != None: - if self.semaphore_values[sem] > 0: - return False - - # check other resources - if not backfill: - # analyse CPU - okcpu = (self.curcpubooked + float(self.cpuperid[tid]) <= self.cpulimit) - # analyse MEM - okmem = (self.curmembooked + float(self.maxmemperid[tid]) <= self.memlimit) - actionlogger.debug ('Condition check --normal-- for ' + str(tid) + ':' + str(self.idtotask[tid]) + ' CPU ' + str(okcpu) + ' MEM ' + str(okmem)) - return (okcpu and okmem) - else: - # only backfill one job at a time - if self.curcpubooked_backfill > 0: - return False - - # not backfilling jobs which either take much memory or use lot's of CPU anyway - # conditions are somewhat arbitrary and can be played with - if float(self.cpuperid[tid]) > 0.9*float(self.args.cpu_limit): - return False - if float(self.maxmemperid[tid])/float(self.args.cpu_limit) >= 1900: - return False - - # analyse CPU - okcpu = (self.curcpubooked_backfill + float(self.cpuperid[tid]) <= self.cpulimit) - okcpu = okcpu and (self.curcpubooked + self.curcpubooked_backfill + float(self.cpuperid[tid]) <= softcpufactor*self.cpulimit) - # analyse MEM - okmem = (self.curmembooked + self.curmembooked_backfill + float(self.maxmemperid[tid]) <= softmemfactor*self.memlimit) - actionlogger.debug ('Condition check --backfill-- for ' + str(tid) + ':' + str(self.idtotask[tid]) + ' CPU ' + str(okcpu) + ' MEM ' + str(okmem)) - return (okcpu and okmem) - return False + return p def ok_to_skip(self, tid): + """ + Decide if task can be skipped based on existence of .log_done + """ done_filename = self.get_done_filename(tid) if os.path.exists(done_filename) and os.path.isfile(done_filename): return True return False - def book_resources(self, tid, backfill = False): - # books the resources used by a certain task - # semaphores - sem = self.workflowspec['stages'][tid].get('semaphore') - if sem != None: - self.semaphore_values[sem]+=1 - - # CPU + MEM - if not backfill: - self.curmembooked+=float(self.maxmemperid[tid]) - self.curcpubooked+=float(self.cpuperid[tid]) - else: - self.curmembooked_backfill+=float(self.maxmemperid[tid]) - self.curcpubooked_backfill+=float(self.cpuperid[tid]) - - def unbook_resources(self, tid, backfill = False): - # "frees" the nominal resources used by a certain task from the accounting - # so that other jobs can be scheduled - sem = self.workflowspec['stages'][tid].get('semaphore') - if sem != None: - self.semaphore_values[sem]-=1 - - # CPU + MEM - if not backfill: - self.curmembooked-=float(self.maxmemperid[tid]) - self.curcpubooked-=float(self.cpuperid[tid]) - else: - self.curmembooked_backfill-=float(self.maxmemperid[tid]) - self.curcpubooked_backfill-=float(self.cpuperid[tid]) - + def try_job_from_candidates(self, taskcandidates, finished): + """ + Try to schedule next tasks - def try_job_from_candidates(self, taskcandidates, process_list, finished): + Args: + taskcandidates: list + list of possible tasks that can be submitted + finished: list + empty list that will be filled with IDs of tasks that were finished in the meantime + """ self.scheduling_iteration = self.scheduling_iteration + 1 # remove "done / skippable" tasks immediately - tasks_skipped = False for tid in taskcandidates.copy(): # <--- the copy is important !! otherwise this loop is not doing what you think if self.ok_to_skip(tid): finished.append(tid) taskcandidates.remove(tid) - tasks_skipped = True actionlogger.info("Skipping task " + str(self.idtotask[tid])) # if tasks_skipped: - # return # ---> we return early in order to preserve some ordering (the next candidate tried should be daughters of skipped jobs) - - # the ordinary process list part - initialcandidates=taskcandidates.copy() - for tid in initialcandidates: + # return # ---> we return early in order to preserve some ordering (the next candidate tried should be daughters of skipped jobs) + # get task ID and proposed niceness from generator + for (tid, nice_value) in self.resource_manager.ok_to_submit(taskcandidates): actionlogger.debug ("trying to submit " + str(tid) + ':' + str(self.idtotask[tid])) - if (len(self.process_list) + len(self.backfill_process_list) < self.max_jobs_parallel) and self.ok_to_submit(tid): - p=self.submit(tid) - if p!=None: - self.book_resources(tid) - self.process_list.append((tid,p)) - taskcandidates.remove(tid) - # minimal delay - time.sleep(0.1) - else: - break #---> we break at first failure assuming some priority (other jobs may come in via backfill) - - # the backfill part for remaining candidates - initialcandidates=taskcandidates.copy() - for tid in initialcandidates: - actionlogger.debug ("trying to backfill submit " + str(tid) + ':' + str(self.idtotask[tid])) - - if (len(self.process_list) + len(self.backfill_process_list) < self.max_jobs_parallel) and self.ok_to_submit(tid, backfill=True): - p=self.submit(tid, 19) - if p!=None: - self.book_resources(tid, backfill=True) - self.process_list.append((tid,p)) - taskcandidates.remove(tid) #-> not sure about this one - # minimal delay - time.sleep(0.1) - else: - continue + if p := self.submit(tid, nice_value): + # explicitly set the nice value here from the process again because it might happen that submit could not change the niceness + # so we let the ResourceManager know what the final niceness is + self.resource_manager.book(tid, p.nice()) + self.process_list.append((tid,p)) + taskcandidates.remove(tid) + # minimal delay + time.sleep(0.1) def stop_pipeline_and_exit(self, process_list): # kill all remaining jobs @@ -811,7 +1105,17 @@ def stop_pipeline_and_exit(self, process_list): exit(1) + def monitor(self, process_list): + """ + Go through all running tasks and get their current resources + + Resources are summed up for tasks and all their children + + Pass CPU, PSS, USS, niceness, current time to metriclogger + + Warn if overall PSS exceeds assigned memory limit + """ self.internalmonitorcounter+=1 if self.internalmonitorcounter % 5 != 0: return @@ -820,10 +1124,10 @@ def monitor(self, process_list): globalCPU=0. globalPSS=0. - globalCPU_backfill=0. - globalPSS_backfill=0. resources_per_task = {} + for tid, proc in process_list: + # proc is Popen object pid=proc.pid if self.pid_to_files.get(pid)==None: @@ -888,24 +1192,33 @@ def monitor(self, process_list): except (psutil.NoSuchProcess, psutil.AccessDenied): pass - resources_per_task[tid]={'iter':self.internalmonitorid, 'name':self.idtotask[tid], 'cpu':totalCPU, 'uss':totalUSS/1024./1024., 'pss':totalPSS/1024./1024, 'nice':self.nicevalues[tid], 'swap':totalSWAP, 'label':self.workflowspec['stages'][tid]['labels']} + time_delta = int((time.perf_counter() - self.start_time) * 1000) + totalUSS = totalUSS / 1024 / 1024 + totalPSS = totalPSS / 1024 / 1024 + nice_value = proc.nice() + resources_per_task[tid]={'iter':self.internalmonitorid, 'name':self.idtotask[tid], 'cpu':totalCPU, 'uss':totalUSS, 'pss':totalPSS, 'nice':nice_value, 'swap':totalSWAP, 'label':self.workflowspec['stages'][tid]['labels']} + self.resource_manager.add_monitored_resources(tid, time_delta, totalCPU / 100, totalPSS) + if nice_value == self.resource_manager.nice_default: + globalCPU += totalCPU + globalPSS += totalPSS + metriclogger.info(resources_per_task[tid]) send_webhook(self.args.webhook, resources_per_task) - - for r in resources_per_task.values(): - if r['nice']==os.nice(0): - globalCPU+=r['cpu'] - globalPSS+=r['pss'] - else: - globalCPU_backfill+=r['cpu'] - globalPSS_backfill+=r['pss'] - - if globalPSS > self.memlimit: + + if globalPSS > self.resource_manager.resource_boundaries.mem_limit: metriclogger.info('*** MEMORY LIMIT PASSED !! ***') # --> We could use this for corrective actions such as killing jobs currently back-filling # (or better hibernating) def waitforany(self, process_list, finished, failingtasks): + """ + Loop through all submitted tasks and check if they are finished + + 1. If process is still running, do nothing + 2. If process is finished, get its return value, update finished and failingtasks lists + 2.1 unbook resources + 2.2 add taken resources and pass the to ResourceManager + """ failuredetected = False failingpids = [] if len(process_list)==0: @@ -920,9 +1233,10 @@ def waitforany(self, process_list, finished, failingtasks): if returncode!=None: actionlogger.info ('Task ' + str(pid) + ' ' + str(tid)+':'+str(self.idtotask[tid]) + ' finished with status ' + str(returncode)) # account for cleared resources - self.unbook_resources(tid, backfill = self.nicevalues[tid]!=os.nice(0) ) + self.resource_manager.unbook(tid) self.procstatus[tid]='Done' finished.append(tid) + #self.validate_resources_running(tid) process_list.remove(p) if returncode != 0: print (str(self.idtotask[tid]) + ' failed ... checking retry') @@ -932,12 +1246,12 @@ def waitforany(self, process_list, finished, failingtasks): actionlogger.info ('Task ' + str(self.idtotask[tid]) + ' failed but marked to be retried ') self.tids_marked_toretry.append(tid) self.retry_counter[tid] += 1 - + else: failuredetected = True failingpids.append(pid) failingtasks.append(tid) - + if failuredetected and self.stoponfailure: actionlogger.info('Stoping pipeline due to failure in stages with PID ' + str(failingpids)) # self.analyse_files_and_connections() @@ -946,26 +1260,16 @@ def waitforany(self, process_list, finished, failingtasks): self.send_checkpoint(failingtasks, self.args.checkpoint_on_failure) self.stop_pipeline_and_exit(process_list) - # empty finished means we have to wait more + # empty finished means we have to wait more return len(finished)==0 - - def get_logfile(self, tid): - # determines the logfile name for this task - taskspec = self.workflowspec['stages'][tid] - taskname = taskspec['name'] - filename = taskname + '.log' - directory = taskspec['cwd'] - return directory + '/' + filename - - def is_worth_retrying(self, tid): # This checks for some signatures in logfiles that indicate that a retry of this task # might have a chance. # Ideally, this should be made user configurable. Either the user could inject a lambda # or a regular expression to use. For now we just put a hard coded list logfile = self.get_logfile(tid) - + return True #! --> for now we just retry tasks a few times # 1) ZMQ_EVENT + interrupted system calls (DPL bug during shutdown) @@ -975,7 +1279,7 @@ def is_worth_retrying(self, tid): # return True # return False - + def cat_logfiles_tostdout(self, taskids): # In case of errors we can cat the logfiles for this taskname @@ -1215,7 +1519,7 @@ def noprogress_errormsg(self): print (msg, file=sys.stderr) def execute(self): - starttime = time.perf_counter() + self.start_time = time.perf_counter() psutil.cpu_percent(interval=None) os.environ['JOBUTILS_SKIPDONE'] = "ON" errorencountered = False @@ -1269,7 +1573,7 @@ def speedup_ROOT_Init(): for i,t in enumerate(self.workflowspec['stages'],0): print (t['name'] + ' (' + str(t['labels']) + ')' + ' ToDo: ' + str(not self.ok_to_skip(i))) exit (0) - + if args.produce_script != None: self.produce_script(args.produce_script) exit (0) @@ -1289,12 +1593,12 @@ def speedup_ROOT_Init(): # ***************** # main control loop # ***************** - currenttimeframe=1 candidates = [ tid for tid in self.possiblenexttask[-1] ] self.process_list=[] # list of tuples of nodes ids and Popen subprocess instances finishedtasks=[] # global list of finished tasks + try: while True: @@ -1306,13 +1610,13 @@ def speedup_ROOT_Init(): finished = [] # --> to account for finished because already done or skipped actionlogger.debug('Sorted current candidates: ' + str([(c,self.idtotask[c]) for c in candidates])) - self.try_job_from_candidates(candidates, self.process_list, finished) + self.try_job_from_candidates(candidates, finished) if len(candidates) > 0 and len(self.process_list) == 0: self.noprogress_errormsg() send_webhook(self.args.webhook,"Unable to make further progress: Quitting") errorencountered = True break - + finished_from_started = [] # to account for finished when actually started failing = [] while self.waitforany(self.process_list, finished_from_started, failing): @@ -1361,10 +1665,10 @@ def speedup_ROOT_Init(): # try to see if this is really a candidate: if self.is_good_candidate(candid, finishedtasks) and candidates.count(candid)==0: candidates.append(candid) - + actionlogger.debug("New candidates " + str( candidates)) send_webhook(self.args.webhook, "New candidates " + str(candidates)) - + if len(candidates)==0 and len(self.process_list)==0: break except Exception as e: @@ -1381,16 +1685,22 @@ def speedup_ROOT_Init(): if errorencountered: statusmsg = "with failures" - print ('\n**** Pipeline done ' + statusmsg + ' (global_runtime : {:.3f}s) *****\n'.format(endtime-starttime)) - actionlogger.debug("global_runtime : {:.3f}s".format(endtime-starttime)) + print ('\n**** Pipeline done ' + statusmsg + ' (global_runtime : {:.3f}s) *****\n'.format(endtime-self.start_time)) + actionlogger.debug("global_runtime : {:.3f}s".format(endtime-self.start_time)) return errorencountered if args.cgroup!=None: myPID=os.getpid() - command="echo " + str(myPID) + " > /sys/fs/cgroup/cpuset/"+args.cgroup+"/tasks" - actionlogger.info("applying cgroups " + command) - os.system(command) + # cgroups such as /sys/fs/cgroup/cpuset//tasks + # or /sys/fs/cgroup/cpu//tasks + command="echo " + str(myPID) + f" > {args.cgroup}" + actionlogger.info(f"Try running in cgroup {args.cgroup}") + waitstatus = os.system(command) + if code := os.waitstatus_to_exitcode(waitstatus): + actionlogger.error(f"Could not apply cgroup") + exit(code) + actionlogger.info("Running in cgroup") executor=WorkflowExecutor(args.workflowfile,jmax=args.maxjobs,args=args) exit (executor.execute()) From 5ce0df9e8b52975d21be82d8b0c858ca86a67535 Mon Sep 17 00:00:00 2001 From: Diego Stocco Date: Tue, 5 Dec 2023 14:34:07 +0100 Subject: [PATCH 0307/1239] Use add_W to build MID bad channels calibration workflow --- DATA/production/calib/mid-badchannels.sh | 38 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/DATA/production/calib/mid-badchannels.sh b/DATA/production/calib/mid-badchannels.sh index 7137d4366..6060b4116 100755 --- a/DATA/production/calib/mid-badchannels.sh +++ b/DATA/production/calib/mid-badchannels.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # shellcheck disable=SC1091 +# Common environment. Notice that this sources common/gen_topo_helper_functions.sh source common/setenv.sh -# shellcheck disable=SC1091 +# Set general arguments source common/getCommonArgs.sh MID_PROXY_INSPEC_EOS="eos:***/INFORMATION" MID_PROXY_INSPEC_DD="dd:FLP/DISTSUBTIMEFRAME/0" MID_RAW_PROXY_INSPEC="A:MID/RAWDATA;$MID_PROXY_INSPEC_DD;$MID_PROXY_INSPEC_EOS" MID_DPL_CHANNEL_CONFIG="name=readout-proxy,type=pull,method=connect,address=ipc://@$INRAWCHANNAME,transport=shmem,rateLogging=1" -CONSUL_ENDPOINT="alio2-cr1-hv-con01.cern.ch:8500" + if [[ -z $CTF_CONFIG ]]; then CTF_CONFIG="--report-data-size-interval 250"; fi if [[ -z $CTF_DIR ]]; then CTF_DIR="$FILEWORKDIR"; fi if [[ -z $CTF_MINSIZE ]]; then CTF_MINSIZE="2000000000"; fi @@ -18,13 +19,32 @@ if [[ -z $CTF_MAX_PER_FILE ]]; then CTF_MAX_PER_FILE="10000"; fi if [[ -z $EPN2EOS_METAFILES_DIR ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi CONFIG_CTF="--output-dir \"$CTF_DIR\" $CTF_CONFIG --output-type ctf --min-file-size ${CTF_MINSIZE} --max-ctf-per-file ${CTF_MAX_PER_FILE} --onlyDet MID $CTF_MAXDETEXT --meta-output-dir $EPN2EOS_METAFILES_DIR" -WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --channel-config \"$MID_DPL_CHANNEL_CONFIG\" | " -WORKFLOW+="o2-mid-raw-to-digits-workflow $ARGS_ALL | " -workflow_has_parameter CTF && WORKFLOW+="o2-mid-entropy-encoder-workflow $ARGS_ALL | o2-ctf-writer-workflow $ARGS_ALL $CONFIG_CTF | " -WORKFLOW+="o2-mid-calibration-workflow $ARGS_ALL | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://alio2-cr1-flp199.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " -workflow_has_parameter QC && WORKFLOW+="o2-qc $ARGS_ALL --config consul-json://${CONSUL_ENDPOINT}/o2/components/qc/ANY/any/mid-calib-qcmn --local --host localhost | " +# CCDB destination for uploads +if [[ -z ${CCDB_POPULATOR_UPLOAD_PATH} ]]; then + if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + CCDB_POPULATOR_UPLOAD_PATH="http://ccdb-test.cern.ch:8080" + CCDB_POPULATOR_UPLOAD_PATH_DCS="$CCDB_POPULATOR_UPLOAD_PATH" + else + CCDB_POPULATOR_UPLOAD_PATH="http://o2-ccdb.internal" + CCDB_POPULATOR_UPLOAD_PATH_DCS="http://alio2-cr1-flp199.cern.ch:8083" + fi +fi +if [[ "${GEN_TOPO_VERBOSE:-}" == "1" ]]; then + echo "CCDB_POPULATOR_UPLOAD_PATH = $CCDB_POPULATOR_UPLOAD_PATH" 1>&2 + echo "CCDB_POPULATOR_UPLOAD_PATH_DCS = $CCDB_POPULATOR_UPLOAD_PATH_DCS" 1>&2 +fi + +WORKFLOW="" +add_W o2-dpl-raw-proxy "--dataspec \"$MID_RAW_PROXY_INSPEC\" --channel-config \"$MID_DPL_CHANNEL_CONFIG\"" "" 0 +add_W o2-mid-raw-to-digits-workflow "" "" 0 +workflow_has_parameter CTF && { + add_W o2-mid-entropy-encoder-workflow "" "" 0 + add_W o2-ctf-writer-workflow "$CONFIG_CTF" "" 0 +} +add_W o2-mid-calibration-workflow "" "" 0 +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDB_POPULATOR_UPLOAD_PATH\" --sspec-min 0 --sspec-max 0" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path=\"$CCDB_POPULATOR_UPLOAD_PATH_DCS\" --sspec-min 1 --sspec-max 1 --name-extention dcs" +workflow_has_parameter QC && add_QC_from_consul "/o2/components/qc/ANY/any/mid-calib-qcmn" "--local --host localhost" WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" if [ "$WORKFLOWMODE" == "print" ]; then From f0be5bbef88a34244fcb0cc7bd55dbe9ba73db7c Mon Sep 17 00:00:00 2001 From: Leonardo Barreto de Oliveira Campos Date: Mon, 15 Jan 2024 04:07:17 -0500 Subject: [PATCH 0308/1239] TRD: New CCDB path for Chamber Status (#1403) * Set new CCDB path for TRD chamber status * Append new path instead of removing old one * Empty, trigger CI --- DATA/production/qc-async/trd.json | 4 ++-- DATA/production/qc-sync/trd.json | 2 +- MC/config/QC/json/trd-digits-task.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index 1b37d231e..d32e1fbfd 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -31,7 +31,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" + "query": "digits:TRD/DIGITS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus" } }, "Tracklets": { @@ -42,7 +42,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" + "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus" } }, "PHTrackMatch": { diff --git a/DATA/production/qc-sync/trd.json b/DATA/production/qc-sync/trd.json index 019762db5..a47867470 100644 --- a/DATA/production/qc-sync/trd.json +++ b/DATA/production/qc-sync/trd.json @@ -159,7 +159,7 @@ "id": "trdall", "active": "true", "machines": [], - "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC", + "query": "digits:TRD/DIGITS/0;tracklets:TRD/TRACKLETS/0;triggers:TRD/TRKTRGRD/0;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus", "samplingConditions": [ { "condition": "random", diff --git a/MC/config/QC/json/trd-digits-task.json b/MC/config/QC/json/trd-digits-task.json index 6b57a85cb..ae05bdfd3 100644 --- a/MC/config/QC/json/trd-digits-task.json +++ b/MC/config/QC/json/trd-digits-task.json @@ -35,7 +35,7 @@ "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC" + "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus" }, "taskParameters": { "peakregionstart": "7.0", @@ -47,4 +47,4 @@ }, "dataSamplingPolicies": [] } -} \ No newline at end of file +} From 23852120482c0e042a4bb89b25770d0eff9f9a6e Mon Sep 17 00:00:00 2001 From: noferini <9963644+noferini@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:09:11 +0100 Subject: [PATCH 0309/1239] track cut tuning for TOF match QC --- DATA/production/qc-async/itstpctof.json | 2 +- DATA/production/qc-async/itstpctofwtrd.json | 2 +- MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json | 4 ++-- .../QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DATA/production/qc-async/itstpctof.json b/DATA/production/qc-async/itstpctof.json index 8b82d6f91..a843da3ea 100644 --- a/DATA/production/qc-async/itstpctof.json +++ b/DATA/production/qc-async/itstpctof.json @@ -39,7 +39,7 @@ "taskParameters" : { "GID" : "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", "verbose" : "false", - "minPtCut" : "0.3f", + "minPtCut" : "0.1f", "etaCut" : "0.8f", "minNTPCClustersCut" : "60", "minDCACut" : "100.f", diff --git a/DATA/production/qc-async/itstpctofwtrd.json b/DATA/production/qc-async/itstpctofwtrd.json index b107730c7..55094c2ad 100644 --- a/DATA/production/qc-async/itstpctofwtrd.json +++ b/DATA/production/qc-async/itstpctofwtrd.json @@ -39,7 +39,7 @@ "taskParameters" : { "GID" : "TPC,TPC-TOF,ITS-TPC,ITS-TPC-TOF,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD,TPC-TRD-TOF", "verbose" : "false", - "minPtCut" : "0.3f", + "minPtCut" : "0.1f", "etaCut" : "0.8f", "minNTPCClustersCut" : "60", "minDCACut" : "100.f", diff --git a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json index e09190776..4d5acc87a 100644 --- a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json @@ -42,7 +42,7 @@ "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF,TPC-TRD-TOF", "verbose": "false", "isMC": "true", - "minPtCut": "0.3f", + "minPtCut": "0.1f", "etaCut": "0.8f", "minNTPCClustersCut": "60", "minDCACut": "100.f", @@ -65,4 +65,4 @@ } }, "dataSamplingPolicies": [] -} \ No newline at end of file +} diff --git a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json index 34f330b28..ef0d21311 100644 --- a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json @@ -43,7 +43,7 @@ "GID": "ITS-TPC,TPC,ITS-TPC-TOF,TPC-TOF", "verbose": "false", "isMC": "true", - "minPtCut": "0.3f", + "minPtCut": "0.1f", "etaCut": "0.8f", "minNTPCClustersCut": "60", "minDCACut": "100.f", @@ -66,4 +66,4 @@ } }, "dataSamplingPolicies": [] -} \ No newline at end of file +} From fbaef945a6f40e1c23096e311ef9995f3e2f2b43 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Mon, 15 Jan 2024 11:28:34 +0100 Subject: [PATCH 0310/1239] Use perVertexProcessing in Pb-Pb This will only work for tags containing the changes of AliceO2Group/AliceO2#12498 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 0818182e9..fedb1f3ba 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -381,7 +381,7 @@ export ITSEXTRAERR="ITSCATrackerParam.sysErrY2[0]=$ERRIB;ITSCATrackerParam.sysEr # ad-hoc options for ITS reco workflow EXTRA_ITSRECO_CONFIG= if [[ $BEAMTYPE == "PbPb" ]]; then - EXTRA_ITSRECO_CONFIG="ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;" + EXTRA_ITSRECO_CONFIG="ITSVertexerParam.clusterContributorsCut=16;ITSVertexerParam.lowMultBeamDistCut=0;ITSCATrackerParam.nROFsPerIterations=12;ITSCATrackerParam.perPrimaryVertexProcessing=true" elif [[ $BEAMTYPE == "pp" ]]; then EXTRA_ITSRECO_CONFIG="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" fi From 098ea85894f4f0cac093a8aff6cd8559499a89cf Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 12 Jan 2024 18:30:59 +0100 Subject: [PATCH 0311/1239] Make TPC reco sensitive to TPCCorrMap param TPCCorrMap is the main way how TPC charge corrections are selected in reconstruction deprecate former options --corrmap-lumi-inst and --corrmap-lumi-mean and introduce --corrmap-lumi-mode --- MC/bin/o2dpg_sim_workflow.py | 14 ++++++++------ UTILS/parse-async-WorkflowConfig.py | 11 ++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index e2eb391cf..efae91015 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -952,14 +952,14 @@ def getDigiTaskName(det): else: tpcclus = createTask(name='tpccluster_' + str(tf), needs=[TPCDigitask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=NWORKERS, mem='2000') tpcclus['cmd'] = '${O2_ROOT}/bin/o2-tpc-chunkeddigit-merger --tpc-lanes ' + str(NWORKERS) - tpcclus['cmd'] += ' | ${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options() + ' --input-type digitizer --output-type clusters,send-clusters-per-sector ' + putConfigValuesNew(["GPU_global","TPCGasParam"],{"GPU_proc.ompThreads" : 1}) + ('',' --disable-mc')[args.no_mc_labels] + tpcclus['cmd'] += ' | ${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options() + ' --input-type digitizer --output-type clusters,send-clusters-per-sector ' + putConfigValuesNew(["GPU_global","TPCGasParam","TPCCorrMap"],{"GPU_proc.ompThreads" : 1}) + ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(tpcclus) tpcreconeeds.append(tpcclus['name']) tpc_corr_scaling_options = anchorConfig.get('tpc-corr-scaling','') TPCRECOtask=createTask(name='tpcreco_'+str(tf), needs=tpcreconeeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], relative_cpu=3/8, mem='16000') TPCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-tpc-reco-workflow ' + getDPL_global_options(bigshm=True) + ' --input-type clusters --output-type tracks,send-clusters-per-sector ' \ - + putConfigValuesNew(["GPU_global","TPCGasParam", "GPU_rec_tpc", "trackTuneParams"], {"GPU_proc.ompThreads":NWORKERS}) + ('',' --disable-mc')[args.no_mc_labels] \ + + putConfigValuesNew(["GPU_global","TPCGasParam", "TPCCorrMap", "GPU_rec_tpc", "trackTuneParams"], {"GPU_proc.ompThreads":NWORKERS}) + ('',' --disable-mc')[args.no_mc_labels] \ + tpc_corr_scaling_options workflow['stages'].append(TPCRECOtask) @@ -980,7 +980,7 @@ def getDigiTaskName(det): ITSTPCMATCHtask=createTask(name='itstpcMatch_'+str(tf), needs=[TPCRECOtask['name'], ITSRECOtask['name'], FT0RECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='8000', relative_cpu=3/8) ITSTPCMATCHtask['cmd'] = '${O2_ROOT}/bin/o2-tpcits-match-workflow ' + getDPL_global_options(bigshm=True) + ' --tpc-track-reader \"tpctracks.root\" --tpc-native-cluster-reader \"--infile tpc-native-clusters.root\" --use-ft0' \ - + putConfigValuesNew(['MFTClustererParam', 'ITSCATrackerParam', 'tpcitsMatch', 'TPCGasParam', 'ITSClustererParam, GPU_rec_tpc, trackTuneParams'], {"NameConf.mDirMatLUT" : ".."}) \ + + putConfigValuesNew(['MFTClustererParam', 'ITSCATrackerParam', 'tpcitsMatch', 'TPCGasParam', 'TPCCorrMap', 'ITSClustererParam, GPU_rec_tpc, trackTuneParams'], {"NameConf.mDirMatLUT" : ".."}) \ + tpc_corr_scaling_options workflow['stages'].append(ITSTPCMATCHtask) @@ -995,7 +995,8 @@ def getDigiTaskName(det): 'ITSCATrackerParam', 'trackTuneParams', 'GPU_rec_tpc', - 'TPCGasParam'], {"NameConf.mDirMatLUT" : ".."}) \ + 'TPCGasParam', + 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) \ + " --track-sources " + anchorConfig.get("o2-trd-global-tracking-options",{}).get("track-sources","all") \ + tpc_corr_scaling_options workflow['stages'].append(TRDTRACKINGtask2) @@ -1013,6 +1014,7 @@ def getDigiTaskName(det): TOFTPCMATCHERtask['cmd'] = '${O2_ROOT}/bin/o2-tof-matcher-workflow ' + getDPL_global_options() \ + putConfigValuesNew(["ITSClustererParam", 'TPCGasParam', + 'TPCCorrMap', 'ITSCATrackerParam', 'MFTClustererParam', 'GPU_rec_tpc', @@ -1163,7 +1165,7 @@ def getDigiTaskName(det): PVFINDERtask = createTask(name='pvfinder_'+str(tf), needs=pvfinderneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=NWORKERS, mem='4000') PVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-primary-vertexing-workflow ' \ - + getDPL_global_options() + putConfigValuesNew(['ITSAlpideParam','MFTAlpideParam', 'pvertexer', 'TPCGasParam'], {"NameConf.mDirMatLUT" : ".."}) + + getDPL_global_options() + putConfigValuesNew(['ITSAlpideParam','MFTAlpideParam', 'pvertexer', 'TPCGasParam', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) PVFINDERtask['cmd'] += ' --vertexing-sources ' + pvfinder_sources + ' --vertex-track-matching-sources ' + pvfinder_matching_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] PVFINDERtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(PVFINDERtask) @@ -1319,7 +1321,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): svfinder_cpu = 8 SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000') SVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-secondary-vertexing-workflow ' - SVFINDERtask['cmd'] += getDPL_global_options(bigshm=True) + svfinder_threads + putConfigValuesNew(['svertexer'], {"NameConf.mDirMatLUT" : ".."}) + SVFINDERtask['cmd'] += getDPL_global_options(bigshm=True) + svfinder_threads + putConfigValuesNew(['svertexer', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) # Take None as default, we only add more if nothing from anchorConfig svfinder_sources = anchorConfig.get("o2-secondary-vertexing-workflow-options",{}).get("vertexing-sources", None) if not svfinder_sources: diff --git a/UTILS/parse-async-WorkflowConfig.py b/UTILS/parse-async-WorkflowConfig.py index 45a4c63df..6252e6c65 100755 --- a/UTILS/parse-async-WorkflowConfig.py +++ b/UTILS/parse-async-WorkflowConfig.py @@ -187,15 +187,12 @@ def parse_important_DPL_args(cmds, flat_config): # itstpc matching if cmd == 'o2-tpcits-match-workflow': corrstring = '' - s1 = extract_args(tokens, '--corrmap-lumi-inst') + s1 = extract_args(tokens, '--lumi-type') if s1: - corrstring += ' --corrmap-lumi-inst ' + s1 - s2 = extract_args(tokens, '--corrmap-lumi-mean') + corrstring += ' --lumi-type ' + s1 + s2 = extract_args(tokens, '--corrmap-lumi-mode') if s2: - corrstring += ' --corrmap-lumi-mean ' + s2 - s3 = extract_args(tokens, '--lumi-type') - if s3: - corrstring += ' --lumi-type ' + s3 + corrstring += ' --corrma-lumi-mode ' + s2 # these are some options applied in multiple places (so save them flatly under tpc-corr-scaling) flat_config['tpc-corr-scaling'] = corrstring From 8f3d906ab4a7b367f339ee2c164b4f94b209b26a Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 17 Jan 2024 17:12:39 +0100 Subject: [PATCH 0312/1239] Change scaling factor to 10^-5 if we run on skimmed data --- .../configurations/asyncReco/setenv_extra.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index fedb1f3ba..209d97cda 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -12,6 +12,11 @@ if [[ $ALIEN_JDL_USEGPUS != 1 ]]; then export DPL_DEFAULT_PIPELINE_LENGTH=16 fi +# check if this is a production on skimmed data +if grep -q /skimmed/ wn.xml ; then + export ON_SKIMMED_DATA=1; +fi + # detector list if [[ -n $ALIEN_JDL_WORKFLOWDETECTORS ]]; then export WORKFLOW_DETECTORS=$ALIEN_JDL_WORKFLOWDETECTORS @@ -561,7 +566,13 @@ if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then if [[ ! -z "$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES" ]]; then export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --enable-unbinned-root-output --sample-unbinned-tsallis --threads 1" fi - if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then + if [[ $ON_SKIMMED_DATA == 1]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then + if [[ $ON_SKIMMED_DATA == 1]] ; then + SAMPLINGFACTORTIMESERIES=0.1f + fi + if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority + export SAMPLINGFACTORTIMESERIES=${ALIEN_JDL_SAMPLINGFACTORTIMESERIES} + fi export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${ALIEN_JDL_SAMPLINGFACTORTIMESERIES}" fi fi From 007ec577d7e41b570a024269aca8d14e14a7c0de Mon Sep 17 00:00:00 2001 From: shahoian Date: Thu, 18 Jan 2024 00:14:48 +0100 Subject: [PATCH 0313/1239] Fix passing some ConfigurableParams to sim_workflow --- MC/bin/o2dpg_sim_workflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index efae91015..273f0590c 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -980,7 +980,7 @@ def getDigiTaskName(det): ITSTPCMATCHtask=createTask(name='itstpcMatch_'+str(tf), needs=[TPCRECOtask['name'], ITSRECOtask['name'], FT0RECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='8000', relative_cpu=3/8) ITSTPCMATCHtask['cmd'] = '${O2_ROOT}/bin/o2-tpcits-match-workflow ' + getDPL_global_options(bigshm=True) + ' --tpc-track-reader \"tpctracks.root\" --tpc-native-cluster-reader \"--infile tpc-native-clusters.root\" --use-ft0' \ - + putConfigValuesNew(['MFTClustererParam', 'ITSCATrackerParam', 'tpcitsMatch', 'TPCGasParam', 'TPCCorrMap', 'ITSClustererParam, GPU_rec_tpc, trackTuneParams'], {"NameConf.mDirMatLUT" : ".."}) \ + + putConfigValuesNew(['MFTClustererParam', 'ITSCATrackerParam', 'tpcitsMatch', 'TPCGasParam', 'TPCCorrMap', 'ITSClustererParam', 'GPU_rec_tpc', 'trackTuneParams', 'ft0tag'], {"NameConf.mDirMatLUT" : ".."}) \ + tpc_corr_scaling_options workflow['stages'].append(ITSTPCMATCHtask) @@ -995,6 +995,7 @@ def getDigiTaskName(det): 'ITSCATrackerParam', 'trackTuneParams', 'GPU_rec_tpc', + 'ft0tag', 'TPCGasParam', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) \ + " --track-sources " + anchorConfig.get("o2-trd-global-tracking-options",{}).get("track-sources","all") \ @@ -1165,7 +1166,7 @@ def getDigiTaskName(det): PVFINDERtask = createTask(name='pvfinder_'+str(tf), needs=pvfinderneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=NWORKERS, mem='4000') PVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-primary-vertexing-workflow ' \ - + getDPL_global_options() + putConfigValuesNew(['ITSAlpideParam','MFTAlpideParam', 'pvertexer', 'TPCGasParam', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) + + getDPL_global_options() + putConfigValuesNew(['ITSAlpideParam','MFTAlpideParam', 'pvertexer', 'TPCGasParam', 'TPCCorrMap', 'ft0tag'], {"NameConf.mDirMatLUT" : ".."}) PVFINDERtask['cmd'] += ' --vertexing-sources ' + pvfinder_sources + ' --vertex-track-matching-sources ' + pvfinder_matching_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] PVFINDERtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(PVFINDERtask) From e9d08bf9af64c75a8fd548c4abe2c8eae821f164 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 17 Jan 2024 16:59:51 +0100 Subject: [PATCH 0314/1239] Forbid AOD merging to fail --- DATA/production/configurations/asyncReco/async_pass.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 71fdb4f5c..c65ac7b49 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -12,7 +12,7 @@ # function to run AOD merging run_AOD_merging() { - o2-aod-merger --input list_$1.list --output $1/AO2D_merged.root > $1/merging.log + o2-aod-merger --input list_$1.list --verbosity 1 --output $1/AO2D_merged.root > $1/merging.log exitcode=$? return $exitcode } @@ -742,7 +742,8 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then if [[ $exitcode -ne 0 ]]; then echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode > validation_error.message echo "Exit code from the process merging DFs inside AO2D for ${aods[$i]} is " $exitcode - echo "As a consequence, we will keep the AO2Ds with unmerged DFs for ${aods[$i]}" + echo "This means that the merging of DFs for ${aods[$i]} FAILED, we make the whole processing FAIL" + exit $exitcode mergedok[$((10#${aods[$i]}))]=0 else echo "Merging of DFs inside the AO2D in ${aods[$i]} worked correctly" @@ -758,7 +759,8 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then AOD_DIR=`dirname $AOD_FILE | sed -e 's|./||'` echo "Inspecting $AOD_DIR:" if [[ ${mergedok[$((10#$AOD_DIR))]} == 0 ]]; then - echo "Merging for $AOD_DIR DID NOT work, we will do nothing for this file" + echo "Merging for $AOD_DIR DID NOT work, we will do nothing for this file - BUT IT SHOULD HAVE NOT HAPPENED, PLEASE CHECK" + exit 8 continue else echo "Merging for $AOD_DIR DID work, let's continue" From 059d746a61236b2c0b5c6d360db1187b8a0dcc2f Mon Sep 17 00:00:00 2001 From: swenzel Date: Thu, 18 Jan 2024 17:44:56 +0100 Subject: [PATCH 0315/1239] load pythia6 library Generator configs accessing TPythia6Decayer now need to load the pythia6 library explicitely, following https://github.com/AliceO2Group/AliceO2/pull/12514 Somehow the EGPythia6.so lib (where TPythia6Decayer is coming from) from ROOT is missing an automatic resolution link to these symbols. --- .../GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C | 1 + .../GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C | 1 + .../GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C | 1 + MC/config/PWGDQ/external/generator/GeneratorParam.C | 2 ++ .../GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C | 1 + .../generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C | 1 + MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C | 1 + 7 files changed, 8 insertions(+) diff --git a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C index 4af9a0c51..341c72844 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C +++ b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToElectronEvtGen_pp13TeV.C @@ -4,6 +4,7 @@ R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/PromptQuarkonia) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorCocktail.C" #include "GeneratorEvtGen.C" diff --git a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C index 4ae78262a..1774ea2c4 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C +++ b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_PbPb5TeV.C @@ -3,6 +3,7 @@ // R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/PromptQuarkonia) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorCocktail.C" #include "GeneratorEvtGen.C" diff --git a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C index 8c267f703..058e3ae36 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C +++ b/MC/config/PWGDQ/external/generator/GeneratorCocktailPromptCharmoniaToMuonEvtGen_pp13TeV.C @@ -3,6 +3,7 @@ // R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/PromptQuarkonia) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorCocktail.C" #include "GeneratorEvtGen.C" diff --git a/MC/config/PWGDQ/external/generator/GeneratorParam.C b/MC/config/PWGDQ/external/generator/GeneratorParam.C index 2f7a25a7f..8218cc2d4 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorParam.C +++ b/MC/config/PWGDQ/external/generator/GeneratorParam.C @@ -1,3 +1,5 @@ +R__LOAD_LIBRARY(libpythia6) + namespace o2 { namespace eventgen diff --git a/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C b/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C index 6875edfa3..2b3a98088 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C +++ b/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C @@ -2,6 +2,7 @@ // o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToElectronEvtGen_pp13TeV()" // R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorEvtGen.C" namespace o2 diff --git a/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C b/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C index cfacf86b3..363453493 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C +++ b/MC/config/PWGDQ/external/generator/GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C @@ -2,6 +2,7 @@ // o2-sim -j 4 -n 10 -g external -o sgn --configKeyValues "GeneratorExternal.fileName=GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV.C;GeneratorExternal.funcName=GeneratorParamPromptJpsiToMuonEvtGen_pp13TeV()" // R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorEvtGen.C" namespace o2 diff --git a/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C index 17f3e75af..a8b7e1e0f 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C +++ b/MC/config/PWGDQ/external/generator/GeneratorPromptCharmonia.C @@ -11,6 +11,7 @@ // R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/EvtGen) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorCocktail.C" #include "GeneratorEvtGen.C" From 900a12874af07852f13593172b8cbfa159fe4113 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Thu, 18 Jan 2024 17:08:34 +0100 Subject: [PATCH 0316/1239] [WF Runner] Fix when initial resources exceed boundaries --- MC/bin/o2_dpg_workflow_runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index cc4ceb57d..b53e5bf63 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -528,7 +528,7 @@ def __init__(self, cpu_limit, mem_limit, dynamic_resources=False, optimistic_res self.mem_limit = mem_limit self.dynamic_resources = dynamic_resources # if this is set, tasks that would normally go beyond the resource limits will tried to be run in any case - self.optimistic_resource = optimistic_resources + self.optimistic_resources = optimistic_resources class TaskResources: @@ -685,6 +685,7 @@ def add_task_resources(self, name, related_tasks_name, cpu, mem, semaphore_strin actionlogger.warning(f"Resource estimates of id {len(self.resources)} overestimates limits, CPU limit: {self.resource_boundaries.cpu_limit}, MEM limit: {self.resource_boundaries.mem_limit}; might not run") if not self.resource_boundaries.optimistic_resources: # exit if we don't dare to try + print(f"Resources of task {name} are exceeding the boundaries.\nCPU: {cpu} (estimate) vs. {self.resource_boundaries.cpu_limit} (boundary)\nMEM: {mem} (estimated) vs. {self.resource_boundaries.mem_limit} (boundary).") exit(1) # or we do dare, let's see what happens... actionlogger.info("We will try to run this task anyway with maximum available resources") From 76b86fa088a2a3d4d4ddbb10a88b4b054f9152a1 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 19 Jan 2024 14:09:48 +0100 Subject: [PATCH 0317/1239] Fix syntax error in setenv_extra.sh --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 209d97cda..8409e7935 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -566,8 +566,8 @@ if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then if [[ ! -z "$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES" ]]; then export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --enable-unbinned-root-output --sample-unbinned-tsallis --threads 1" fi - if [[ $ON_SKIMMED_DATA == 1]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then - if [[ $ON_SKIMMED_DATA == 1]] ; then + if [[ $ON_SKIMMED_DATA == 1 ]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then + if [[ $ON_SKIMMED_DATA == 1 ]] ; then SAMPLINGFACTORTIMESERIES=0.1f fi if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority From b52d2cd2b836085df533d38a4730316cebb2c06d Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 22 Jan 2024 16:30:54 +0100 Subject: [PATCH 0318/1239] [SimWF] Always set a process for Pythia8 (#1416) Co-authored-by: Benedikt Volkel --- MC/bin/o2dpg_sim_workflow.py | 6 +++--- MC/config/common/pythia8/utils/mkpy8cfg.py | 21 +++++++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 273f0590c..19a2e9b9a 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -55,7 +55,7 @@ parser.add_argument('--dump-config',help="Dump JSON file with all settings used in workflow", default='user_config.json') parser.add_argument('-ns',help='number of signal events / timeframe', default=20) parser.add_argument('-gen',help='generator: pythia8, extgen', default='') -parser.add_argument('-proc',help='process type: inel, dirgamma, jets, ccbar, ...', default='') +parser.add_argument('-proc',help='process type: inel, dirgamma, jets, ccbar, ...', default='none') parser.add_argument('-trigger',help='event selection: particle, external', default='') parser.add_argument('-ini',help='generator init parameters file (full paths required), for example: ${O2DPG_ROOT}/MC/config/PWGHF/ini/GeneratorHF.ini', default='') parser.add_argument('-confKey',help='generator or trigger configuration key values, for example: "GeneratorPythia8.config=pythia8.cfg;A.x=y"', default='') @@ -409,7 +409,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # Background PYTHIA configuration BKG_CONFIG_task=createTask(name='genbkgconf') BKG_CONFIG_task['cmd'] = 'echo "placeholder / dummy task"' - if GENBKG == 'pythia8' and PROCESSBKG != "": + if GENBKG == 'pythia8': print('Background generator seed: ', SIMSEED) BKG_CONFIG_task['cmd'] = '${O2DPG_ROOT}/MC/config/common/pythia8/utils/mkpy8cfg.py \ --output=pythia8bkg.cfg \ @@ -626,7 +626,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # produce the signal configuration SGN_CONFIG_task=createTask(name='gensgnconf_'+str(tf), tf=tf, cwd=timeframeworkdir) SGN_CONFIG_task['cmd'] = 'echo "placeholder / dummy task"' - if GENERATOR == 'pythia8' and PROCESS!='': + if GENERATOR == 'pythia8': SGN_CONFIG_task['cmd'] = '${O2DPG_ROOT}/MC/config/common/pythia8/utils/mkpy8cfg.py \ --output=pythia8.cfg \ --seed='+str(TFSEED)+' \ diff --git a/MC/config/common/pythia8/utils/mkpy8cfg.py b/MC/config/common/pythia8/utils/mkpy8cfg.py index d7530885f..460af1e99 100755 --- a/MC/config/common/pythia8/utils/mkpy8cfg.py +++ b/MC/config/common/pythia8/utils/mkpy8cfg.py @@ -5,6 +5,20 @@ import argparse + +def derive_process(process_in, beam_type_a, beam_type_b): + """ + Derive the process from the beam types if not already set. + """ + if process_in != "none": + return process_in + if beam_type_a == 2212 and beam_type_b == 2212: + return "inel" + if beam_type_a == 1000822080 and beam_type_b == 1000822080: + return "heavy_ion" + return "none" + + parser = argparse.ArgumentParser(description='Make Pythia8 configuration', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -52,6 +66,9 @@ args = parser.parse_args() +# derive the process +args.process = derive_process(args.process, args.idA, args.idB) + ### open output file fout = open(args.output, 'w') @@ -146,7 +163,7 @@ fout.write('### <-- generated by mkpy8cfg.py \n') fout.write('\n') - + ### appended files if args.append is not None : for i in args.append : @@ -159,7 +176,7 @@ fout.write('\n') fout.write('### <-- included from %s \n' % (i)) fout.write('\n') - + ### user commands if args.command is not None : fout.write('### --> user commands \n') From 5a2f80c7fb9476918259233ffa892f45dd70142f Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Mon, 22 Jan 2024 17:21:46 +0100 Subject: [PATCH 0319/1239] Update certificate handling in runGRIDContainerized.sh --- GRID/utils/runGRIDContainerized.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GRID/utils/runGRIDContainerized.sh b/GRID/utils/runGRIDContainerized.sh index 6165a128f..ca98f616c 100755 --- a/GRID/utils/runGRIDContainerized.sh +++ b/GRID/utils/runGRIDContainerized.sh @@ -24,11 +24,11 @@ ALIEN_KEYFILE=$(ls -t /tmp/tokenkey_*.pem 2> /dev/null | head -n 1) [ "${ALIEN_CERTFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 [ "${ALIEN_KEYFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 -JALIEN_TOKEN_CERT=$(awk '! /^-/{print $0}' ${ALIEN_CERTFILE} | tr -d '\n') -JALIEN_TOKEN_KEY=$(awk '! /^-/{print $0}' ${ALIEN_KEYFILE} | tr -d '\n') +cp ${ALIEN_CERTFILE} ${WORK_DIR}/usercert.pem +cp ${ALIEN_KEYFILE} ${WORK_DIR}/userkey.pem -echo "JALIEN_TOKEN_CERT=${JALIEN_TOKEN_CERT}" > ${WORK_DIR}/envfile -echo "JALIEN_TOKEN_KEY=${JALIEN_TOKEN_KEY}" >> ${WORK_DIR}/envfile +echo "JALIEN_TOKEN_CERT=/workdir/usercert.pem" > ${WORK_DIR}/envfile +echo "JALIEN_TOKEN_KEY=/workdir/userkey.pem" >> ${WORK_DIR}/envfile # launch job = script inside the container in the workdir /cvmfs/alice.cern.ch/containers/bin/apptainer/current/bin/apptainer exec -C -B /cvmfs:/cvmfs,${WORK_DIR}:/workdir \ From 790081a9f5b7912ca4d75b4dabe97ac930a877e8 Mon Sep 17 00:00:00 2001 From: catalinristea Date: Mon, 22 Jan 2024 18:35:45 +0200 Subject: [PATCH 0320/1239] new anchoredMC.sh for nowadays productions (#1340) * prevent from sourcing the script * stop immediately if running async_pass.sh fails --> removed the condiction on "NO_MC" * add some comments * remove test and development-related code * unify configuration * consistently use CCDB cache, make setenv_extra.sh executable * add --help * remove -proc heavy_ion * add possibility to disable QC (e.g. disable with DISABLE_QC=1) --------- Co-authored-by: Catalin Ristea Co-authored-by: Benedikt Volkel --- MC/run/ANCHOR/anchorMC.sh | 269 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100755 MC/run/ANCHOR/anchorMC.sh diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh new file mode 100755 index 000000000..0892bf906 --- /dev/null +++ b/MC/run/ANCHOR/anchorMC.sh @@ -0,0 +1,269 @@ +#!/bin/bash + +# add distortion maps +# https://alice.its.cern.ch/jira/browse/O2-3346?focusedCommentId=300982&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-300982 +# +# export O2DPG_ENABLE_TPC_DISTORTIONS=ON +# SCFile=$PWD/distortions_5kG_lowIR.root # file needs to be downloaded +# export O2DPG_TPC_DIGIT_EXTRA=" --distortionType 2 --readSpaceCharge ${SCFile} " + +# +# procedure setting up and executing an anchored MC +# + +######################## +# helper functionality # +######################## + +print_help() +{ + echo "Usage: ./anchorMC.sh" + echo + echo "This needs O2 and O2DPG loaded from alienv." + echo + echo "Make sure the following env variables are set:" + echo "ALIEN_JDL_LPMANCHORPASSNAME or ANCHORPASSNAME," + echo "ALIEN_JDL_MCANCHOR or MCANCHOR," + echo "ALIEN_JDL_LPMPASSNAME or PASSNAME," + echo "ALIEN_JDL_LPMRUNNUMBER or RUNNUMBER," + echo "ALIEN_JDL_LPMPRODUCTIONTYPE or PRODUCTIONTYPE," + echo "ALIEN_JDL_LPMINTERACTIONTYPE or INTERACTIONTYPE," + echo "ALIEN_JDL_LPMPRODUCTIONTAG or PRODUCTIONTAG," + echo "ALIEN_JDL_LPMANCHORRUN or ANCHORRUN," + echo "ALIEN_JDL_LPMANCHORPRODUCTION or ANCHORPRODUCTION," + echo "ALIEN_JDL_LPMANCHORYEAR or ANCHORYEAR," + echo + echo "as well as:" + echo "NTIMEFRAMES," + echo "NSIGEVENTS," + echo "SPLITID," + echo "CYCLE," + echo "PRODSPLIT." + echo + echo "Optional are:" + echo "NWORKERS," + echo "ALIEN_JDL_CPULIMIT or CPULIMIT," + echo "ALIEN_JDL_SIMENGINE or SIMENGINE." + echo "DISABLE_QC (set this to disable QC, e.g. DISABLE_QC=1)" +} + +# Prevent the script from being soured to omit unexpected surprises when exit is used +SCRIPT_NAME="$(basename "$(test -L "$0" && readlink "$0" || echo "$0")")" +if [ "${SCRIPT_NAME}" != "$(basename ${BASH_SOURCE[0]})" ] ; then + echo "ERROR: This script cannot not be sourced" >&2 + return 1 +fi + +while [ "$1" != "" ] ; do + case $1 in + --help|-h ) shift + print_help + exit 0 + ;; + * ) echo "Unknown argument ${1}" + exit 1 + ;; + esac +done + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +################################################################# +# Set all required variables to identify an anchored production # +################################################################# + +# Allow for both "ALIEN_JDL_LPM" as well as "KEY" + +# the only two where there is a real default for +export ALIEN_JDL_CPULIMIT=${ALIEN_JDL_CPULIMIT:-${CPULIMIT:-8}} +export ALIEN_JDL_SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE:-TGeant4}} +# all others MUST be set by the user/on the outside +export ALIEN_JDL_LPMANCHORPASSNAME=${ALIEN_JDL_LPMANCHORPASSNAME:-${ANCHORPASSNAME}} +export ALIEN_JDL_MCANCHOR=${ALIEN_JDL_MCANCHOR:-${MCANCHOR}} +export ALIEN_JDL_LPMPASSNAME=${ALIEN_JDL_LPMPASSNAME:-${PASSNAME}} +export ALIEN_JDL_LPMRUNNUMBER=${ALIEN_JDL_LPMRUNNUMBER:-${RUNNUMBER}} +export ALIEN_JDL_LPMPRODUCTIONTYPE=${ALIEN_JDL_LPMPRODUCTIONTYPE:-${PRODUCTIONTYPE}} +export ALIEN_JDL_LPMINTERACTIONTYPE=${ALIEN_JDL_LPMINTERACTIONTYPE:-${INTERACTIONTYPE}} +export ALIEN_JDL_LPMPRODUCTIONTAG=${ALIEN_JDL_LPMPRODUCTIONTAG:-${PRODUCTIONTAG}} +export ALIEN_JDL_LPMANCHORRUN=${ALIEN_JDL_LPMANCHORRUN:-${ANCHORRUN}} +export ALIEN_JDL_LPMANCHORPRODUCTION=${ALIEN_JDL_LPMANCHORPRODUCTION:-${ANCHORPRODUCTION}} +export ALIEN_JDL_LPMANCHORYEAR=${ALIEN_JDL_LPMANCHORYEAR:-${ANCHORYEAR}} + +# cache the production tag, will be set to a special anchor tag; reset later in fact +ALIEN_JDL_LPMPRODUCTIONTAG_KEEP=$ALIEN_JDL_LPMPRODUCTIONTAG +echo "Substituting ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG with ALIEN_JDL_LPMANCHORPRODUCTION=$ALIEN_JDL_LPMANCHORPRODUCTION for simulating reco pass..." +ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMANCHORPRODUCTION + +# ZDC causes issues for sim +#export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,ZDC,CTP +export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP + +# check variables that need to be set +[ -z "${ALIEN_JDL_LPMANCHORPASSNAME}" ] && { echo "ERROR: Set ALIEN_JDL_LPMANCHORPASSNAME or ANCHORPASSNAME" ; exit 1 ; } +[ -z "${ALIEN_JDL_MCANCHOR}" ] && { echo "ERROR: Set ALIEN_JDL_MCANCHOR or MCANCHOR" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMPASSNAME}" ] && { echo "ERROR: Set ALIEN_JDL_LPMPASSNAME or PASSNAME" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMRUNNUMBER}" ] && { echo "ERROR: Set ALIEN_JDL_LPMRUNNUMBER or RUNNUMBER" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMPRODUCTIONTYPE}" ] && { echo "ERROR: Set ALIEN_JDL_LPMPRODUCTIONTYPE or PRODUCTIONTYPE" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMINTERACTIONTYPE}" ] && { echo "ERROR: Set ALIEN_JDL_LPMINTERACTIONTYPE or INTERACTIONTYPE" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMPRODUCTIONTAG}" ] && { echo "ERROR: Set ALIEN_JDL_LPMPRODUCTIONTAG or PRODUCTIONTAG" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMANCHORRUN}" ] && { echo "ERROR: Set ALIEN_JDL_LPMANCHORRUN or ANCHORRUN" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMANCHORPRODUCTION}" ] && { echo "ERROR: Set ALIEN_JDL_LPMANCHORPRODUCTION or ANCHORPRODUCTION" ; exit 1 ; } +[ -z "${ALIEN_JDL_LPMANCHORYEAR}" ] && { echo "ERROR: Set ALIEN_JDL_LPMANCHORYEAR or ANCHORYEAR" ; exit 1 ; } + +[ -z "${NTIMEFRAMES}" ] && { echo "ERROR: Set NTIMEFRAMES" ; exit 1 ; } +[ -z "${NSIGEVENTS}" ] && { echo "ERROR: Set NSIGEVENTS" ; exit 1 ; } +[ -z "${SPLITID}" ] && { echo "ERROR: Set SPLITID" ; exit 1 ; } +[ -z "${CYCLE}" ] && { echo "ERROR: Set CYCLE" ; exit 1 ; } +[ -z "${PRODSPLIT}" ] && { echo "ERROR: Set PRODSPLIT" ; exit 1 ; } + +# also for this keep a real default +NWORKERS=${NWORKERS:-8} + + +# default async_pass.sh script +DPGRECO=$O2DPG_ROOT/DATA/production/configurations/asyncReco/async_pass.sh +# default destenv_extra.sh script +DPGSETENV=$O2DPG_ROOT/DATA/production/configurations/asyncReco/setenv_extra.sh + +# a specific async_pass.sh script is in the current directory, assume that one should be used +if [[ -f async_pass.sh ]]; then + # the default is executable, however, this may not be, so make it so + chmod +x async_pass.sh + DPGRECO=./async_pass.sh +else + cp -v $DPGRECO . +fi + +# if there is no setenv_extra.sh in this directory (so no special version is "shipped" with this rpodcution), copy the default one +if [[ ! -f setenv_extra.sh ]] ; then + cp ${DPGSETENV} . + echo "[INFO alien_setenv_extra.sh] Use default setenv_extra.sh from ${DPGSETENV}." +else + echo "[INFO alien_setenv_extra.sh] setenv_extra.sh was found in the current working directory, use it." +fi + +chmod u+x setenv_extra.sh + +echo "[INFO alien_async_pass.sh] Setting up DPGRECO to ${DPGRECO}" + +# settings that are MC-specific, modify setenv_extra.sh in-place +sed -i 's/GPU_global.dEdxUseFullGainMap=1;GPU_global.dEdxDisableResidualGainMap=1/GPU_global.dEdxSplineTopologyCorrFile=splines_for_dedx_V1_MC_iter0_PP.root;GPU_global.dEdxDisableTopologyPol=1;GPU_global.dEdxDisableGainMap=1;GPU_global.dEdxDisableResidualGainMap=1;GPU_global.dEdxDisableResidualGain=1/' setenv_extra.sh +### ??? + +# take out line running the workflow (if we don't have data input) +[ ${CTF_TEST_FILE} ] || sed -i '/WORKFLOWMODE=run/d' async_pass.sh + +# create workflow ---> creates the file that can be parsed +export IGNORE_EXISTING_SHMFILES=1 +touch list.list + +# run the async_pass.sh and store output to log file for later inspection and extraction of information +./async_pass.sh ${CTF_TEST_FILE:-""} 2&> async_pass_log.log +RECO_RC=$? + +echo "RECO finished with ${RECO_RC}" + +if [[ "${RECO_RC}" != "0" ]] ; then + exit ${RECO_RC} +fi + +ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG_KEEP +echo "Setting back ALIEN_JDL_LPMPRODUCTIONTAG to $ALIEN_JDL_LPMPRODUCTIONTAG" + +# now create the local MC config file --> config-config.json +${O2DPG_ROOT}/UTILS/parse-async-WorkflowConfig.py +ASYNC_WF_RC=${?} + +# check if config reasonably created +if [[ "${ASYNC_WF_RC}" != "0" || `grep "o2-ctf-reader-workflow-options" config-json.json 2> /dev/null | wc -l` == "0" ]]; then + echo "Problem in anchor config creation. Exiting." + exit 1 +fi + +# -- CREATE THE MC JOB DESCRIPTION ANCHORED TO RUN -- + +MODULES="--skipModules ZDC" +# introduce variable to make usage clear +SEED=${ALIEN_PROC_ID} +# Since this is used, set it explicitly +ALICEO2_CCDB_LOCALCACHE=${ALICEO2_CCDB_LOCALCACHE:-$(pwd)/ccdb} + +# these arguments will be digested by o2dpg_sim_workflow_anchored.py +baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}" + +# these arguments will be passed as well but only evetually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py +remainingargs="-gen pythia8 -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" +remainingargs="${remainingargs} -e ${ALIEN_JDL_SIMENGINE} -j ${NWORKERS}" +remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" +remainingargs="${remainingargs} --anchor-config config-json.json" + +echo "baseargs: ${baseargs}" +echo "remainingargs: ${remainingargs}" + +# query CCDB has changed, w/o "_" +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow_anchored.py ${baseargs} -- ${remainingargs} &> timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log +if [ "$?" != "0" ] ; then + echo "Problem during anchor timestamp sampling and workflow creation. Exiting." + exit 1 +fi + +TIMESTAMP=`grep "Determined timestamp to be" timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log | awk '//{print $6}'` +echo "TIMESTAMP IS ${TIMESTAMP}" + +# -- PREFETCH CCDB OBJECTS TO DISC -- +# (make sure the right objects at the right timestamp are fetched +# until https://alice.its.cern.ch/jira/browse/O2-2852 is fixed) +# NOTE: In fact, looking at the ticket, it should be fixed now. However, not changing at the moment as further tests would be needed to confirm here. + +CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /FV0/Calibration/ChannelTimeOffset" + +${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS} -d ${ALICEO2_CCDB_LOCALCACHE} --timestamp ${TIMESTAMP} +if [ ! "$?" == "0" ]; then + echo "Problem during CCDB prefetching of ${CCDBOBJECTS}. Exiting." + exit 1 +fi + +# -- Create aligned geometry using ITS and MFT ideal alignments to avoid overlaps in geant +CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align MFT/Calib/Align" +TIMESTAMP_IDEAL_MC=1 +${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS_IDEAL_MC} -d ${ALICEO2_CCDB_LOCALCACHE} --timestamp ${TIMESTAMP_IDEAL_MC} +if [ ! "$?" == "0" ]; then + echo "Problem during CCDB prefetching of ${CCDBOBJECTS_IDEAL_MC}. Exiting." + exit 1 +fi + +echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align,MFT/Calib/Align -b +mkdir -p $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned +ln -s -f $PWD/o2sim_geometry-aligned.root $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned/snapshot.root + +# -- RUN THE MC WORKLOAD TO PRODUCE AOD -- + +export FAIRMQ_IPC_PREFIX=./ + +echo "Ready to start main workflow" + +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} +MCRC=$? # <--- we'll report back this code + +[[ ! -z "${DISABLE_QC}" ]] && echo "INFO: QC is disabled, skip it." + +if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then + # do QC tasks + echo "Doing QC" + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k + MCRC=$? +fi + +# +# full logs tar-ed for output, regardless the error code or validation - to catch also QC logs... +# +if [[ -n "$ALIEN_PROC_ID" ]]; then + find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "pythia8.cfg" \) | tar -czvf debug_log_archive.tgz -T - + find ./ \( -name "*.log*" -o -name "*mergerlog*" -o -name "*serverlog*" -o -name "*workerlog*" -o -name "*.root" \) | tar -czvf debug_full_archive.tgz -T - +fi + +unset FAIRMQ_IPC_PREFIX + +exit ${MCRC} From 2fb1e8da18a4f848c84e93f8809d4944b14db35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Tue, 23 Jan 2024 12:21:31 +0100 Subject: [PATCH 0321/1239] TOF: align analysis QC (#1417) - format analysis QC jsons - Add multiplicity table - Add good match cut for TOF QC --- .../pbpb/analysis-testing-data.json | 240 ++++----- .../json/analyses_config.json | 2 + .../default/pbpb/analysis-testing-data.json | 250 +++++----- .../default/pbpb/analysis-testing-mc.json | 464 +++++++++--------- .../default/pp/analysis-testing-data.json | 240 ++++----- .../json/default/pp/analysis-testing-mc.json | 464 +++++++++--------- .../analysis_testing/json/format_json.sh | 2 +- 7 files changed, 837 insertions(+), 825 deletions(-) diff --git a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json index d71d7a628..5686e51b5 100644 --- a/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/EventSelectionQA/pbpb/analysis-testing-data.json @@ -123,8 +123,8 @@ "useWeightExotic": "0" }, "event-selection-qa-task": { + "isLowFlux": "false", "isMC": "false", - "isLowFlux" : "false", "processRun2": "false", "processRun3": "true" }, @@ -294,8 +294,18 @@ "rapidity": "0.5", "v0cospa": "0.995" }, - "tof-signal": "", - "tpc-pid-qa": { + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "1", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", "etaBins": { "values": [ 50, @@ -303,43 +313,52 @@ 1 ] }, - "dEdxBins": { + "expSigmaBins": { "values": [ - 3000, + 200, 0, - 1000 + 200 + ] + }, + "logAxis": "0", + "maxP": "5", + "minP": "0.1", + "nBinsP": "400", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 ] }, - "applyRapidityCut": "0", - "processFullWithTOFDeuteron": "0", "phiBins": { "values": [ - 50, + 25, 0, 6.283185307179586 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "1", - "processKaon": "0", "processAlpha": "0", - "processFullWithTOFPion": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "0", "processFullHelium3": "1", - "minP": "0.01", - "enableDeDxPlot": "1", - "processFullWithTOFElectron": "0", - "logAxis": "1", - "processFullWithTOFAlpha": "0", "processFullKaon": "1", + "processFullMuon": "0", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processHelium3": "0", + "processKaon": "0", "processMuon": "0", - "processFullDeuteron": "1", - "processFullWithTOFMuon": "0", - "processFullWithTOFProton": "0", - "maxP": "5", - "processFullAlpha": "0", - "processElectron": "0", - "processDeuteron": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "produceDeltaTEtaPhiMap": "0", + "ptDeltaTEtaPhiMap": "3", + "splitSignalPerCharge": "1", "trackLengthBins": { "values": [ 50, @@ -347,33 +366,42 @@ 1000 ] }, - "processFullProton": "1", - "processFullWithTOFKaon": "0", - "minTPCNcls": "0", - "processFullMuon": "1", - "processFullWithTOFTriton": "0", - "nSigmaBins": { - "values": [ - 400, - -10, - 10 - ] - }, - "trackSelection": "1", - "processFullWithTOFHelium3": "0", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "trackSelection": "1" + }, + "tof-signal": "", + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "0", + "dEdxBins": { "values": [ - 100, + 3000, 0, - 200 + 1000 ] }, - "applyEvSel": "2", - "nBinsP": "400", "deltaBins": { "values": [ 100, @@ -381,9 +409,7 @@ 1000 ] }, - "processFullPion": "1" - }, - "tof-pid-qa": { + "enableDeDxPlot": "1", "etaBins": { "values": [ 50, @@ -391,42 +417,18 @@ 1 ] }, - "applyRapidityCut": "1", - "phiBins": { + "expSigmaBins": { "values": [ - 25, + 100, 0, - 6.283185307179586 + 200 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "0", - "processKaon": "0", - "processAlpha": "0", - "processFullHelium3": "1", - "minP": "0.1", - "logAxis": "0", - "processFullKaon": "1", - "produceDeltaTEtaPhiMap": "0", - "processMuon": "0", - "processFullDeuteron": "1", + "logAxis": "1", "maxP": "5", - "processFullAlpha": "0", - "ptDeltaTEtaPhiMap": "3", - "processElectron": "0", - "processDeuteron": "0", - "trackLengthBins": { - "values": [ - 50, - 0, - 1000 - ] - }, - "processFullProton": "1", - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", - "processFullMuon": "0", + "minP": "0.01", + "minTPCNcls": "0", + "nBinsP": "400", "nSigmaBins": { "values": [ 400, @@ -434,51 +436,49 @@ 10 ] }, - "trackSelection": "1", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "phiBins": { "values": [ - 200, + 50, 0, - 200 + 6.283185307179586 ] }, - "applyEvSel": "2", - "nBinsP": "400", - "deltaBins": { + "processAlpha": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "1", + "processFullHelium3": "1", + "processFullKaon": "1", + "processFullMuon": "1", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processFullWithTOFAlpha": "0", + "processFullWithTOFDeuteron": "0", + "processFullWithTOFElectron": "0", + "processFullWithTOFHelium3": "0", + "processFullWithTOFKaon": "0", + "processFullWithTOFMuon": "0", + "processFullWithTOFPion": "0", + "processFullWithTOFProton": "0", + "processFullWithTOFTriton": "0", + "processHelium3": "0", + "processKaon": "0", + "processMuon": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "splitSignalPerCharge": "1", + "trackLengthBins": { "values": [ - 100, - -1000, + 50, + 0, 1000 ] }, - "processFullPion": "1" - }, - "tpc-pid-full": { - "autofetchNetworks": "true", - "ccdb-timestamp": "0", - "ccdb-url": "http://alice-ccdb.cern.ch", - "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "true", - "networkPathCCDB": "Analysis/PID/TPC/ML", - "networkPathLocally": "network.onnx", - "networkSetNumThreads": "0", - "param-file": "", - "pid-al": "-1", - "pid-de": "-1", - "pid-el": "-1", - "pid-he": "-1", - "pid-ka": "-1", - "pid-mu": "-1", - "pid-pi": "-1", - "pid-pr": "-1", - "pid-tr": "-1", - "recoPass": "", - "skipTPCOnly": "false", - "useNetworkCorrection": "false" + "trackSelection": "1" }, "track-extension": { "processRun2": "false", diff --git a/MC/config/analysis_testing/json/analyses_config.json b/MC/config/analysis_testing/json/analyses_config.json index 9c060eaa1..e2721c661 100644 --- a/MC/config/analysis_testing/json/analyses_config.json +++ b/MC/config/analysis_testing/json/analyses_config.json @@ -43,6 +43,7 @@ "tasks": ["o2-analysis-lf-lambdakzerobuilder", "o2-analysis-track-propagation", "o2-analysis-trackselection", + "o2-analysis-multiplicity-table", "o2-analysis-pid-tof-base", "o2-analysis-pid-tof", "o2-analysis-pid-tof-full", @@ -75,6 +76,7 @@ "o2-analysis-track-propagation", "o2-analysis-trackselection", "o2-analysis-event-selection", + "o2-analysis-multiplicity-table", "o2-analysis-pid-tof-base", "o2-analysis-pid-tof", "o2-analysis-pid-tof-full", diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json index a1c192b40..69a6cd245 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json @@ -293,8 +293,19 @@ "rapidity": "0.5", "v0cospa": "0.995" }, - "tof-signal": "", - "tpc-pid-qa": { + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "1", + "requireGoodMatchTracks": "true", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", "etaBins": { "values": [ 50, @@ -302,43 +313,52 @@ 1 ] }, - "dEdxBins": { + "expSigmaBins": { "values": [ - 3000, + 200, 0, - 1000 + 200 + ] + }, + "logAxis": "0", + "maxP": "5", + "minP": "0.1", + "nBinsP": "400", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 ] }, - "applyRapidityCut": "0", - "processFullWithTOFDeuteron": "0", "phiBins": { "values": [ - 50, + 25, 0, 6.283185307179586 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "1", - "processKaon": "0", "processAlpha": "0", - "processFullWithTOFPion": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "0", "processFullHelium3": "1", - "minP": "0.01", - "enableDeDxPlot": "1", - "processFullWithTOFElectron": "0", - "logAxis": "1", - "processFullWithTOFAlpha": "0", "processFullKaon": "1", + "processFullMuon": "0", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processHelium3": "0", + "processKaon": "0", "processMuon": "0", - "processFullDeuteron": "1", - "processFullWithTOFMuon": "0", - "processFullWithTOFProton": "0", - "maxP": "5", - "processFullAlpha": "0", - "processElectron": "0", - "processDeuteron": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "produceDeltaTEtaPhiMap": "0", + "ptDeltaTEtaPhiMap": "3", + "splitSignalPerCharge": "1", "trackLengthBins": { "values": [ 50, @@ -346,33 +366,51 @@ 1000 ] }, - "processFullProton": "1", - "processFullWithTOFKaon": "0", - "minTPCNcls": "0", - "processFullMuon": "1", - "processFullWithTOFTriton": "0", - "nSigmaBins": { - "values": [ - 400, - -10, - 10 - ] - }, - "trackSelection": "1", - "processFullWithTOFHelium3": "0", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "trackSelection": "1" + }, + "tof-signal": { + "ccdb-timestamp": "-1", + "ccdb-url": "http://alice-ccdb.cern.ch", + "distanceForGoodMatch": "3", + "distanceForGoodMatchLowMult": "999", + "multThreshold": "50", + "processRun2": "false", + "processRun3": "true", + "timeShiftCCDBPath": "" + }, + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "0", + "dEdxBins": { "values": [ - 100, + 3000, 0, - 200 + 1000 ] }, - "applyEvSel": "2", - "nBinsP": "400", "deltaBins": { "values": [ 100, @@ -380,9 +418,7 @@ 1000 ] }, - "processFullPion": "1" - }, - "tof-pid-qa": { + "enableDeDxPlot": "1", "etaBins": { "values": [ 50, @@ -390,42 +426,18 @@ 1 ] }, - "applyRapidityCut": "1", - "phiBins": { + "expSigmaBins": { "values": [ - 25, + 100, 0, - 6.283185307179586 + 200 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "0", - "processKaon": "0", - "processAlpha": "0", - "processFullHelium3": "1", - "minP": "0.1", - "logAxis": "0", - "processFullKaon": "1", - "produceDeltaTEtaPhiMap": "0", - "processMuon": "0", - "processFullDeuteron": "1", + "logAxis": "1", "maxP": "5", - "processFullAlpha": "0", - "ptDeltaTEtaPhiMap": "3", - "processElectron": "0", - "processDeuteron": "0", - "trackLengthBins": { - "values": [ - 50, - 0, - 1000 - ] - }, - "processFullProton": "1", - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", - "processFullMuon": "0", + "minP": "0.01", + "minTPCNcls": "0", + "nBinsP": "400", "nSigmaBins": { "values": [ 400, @@ -433,51 +445,49 @@ 10 ] }, - "trackSelection": "1", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "phiBins": { "values": [ - 200, + 50, 0, - 200 + 6.283185307179586 ] }, - "applyEvSel": "2", - "nBinsP": "400", - "deltaBins": { + "processAlpha": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "1", + "processFullHelium3": "1", + "processFullKaon": "1", + "processFullMuon": "1", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processFullWithTOFAlpha": "0", + "processFullWithTOFDeuteron": "0", + "processFullWithTOFElectron": "0", + "processFullWithTOFHelium3": "0", + "processFullWithTOFKaon": "0", + "processFullWithTOFMuon": "0", + "processFullWithTOFPion": "0", + "processFullWithTOFProton": "0", + "processFullWithTOFTriton": "0", + "processHelium3": "0", + "processKaon": "0", + "processMuon": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "splitSignalPerCharge": "1", + "trackLengthBins": { "values": [ - 100, - -1000, + 50, + 0, 1000 ] }, - "processFullPion": "1" - }, - "tpc-pid-full": { - "autofetchNetworks": "true", - "ccdb-timestamp": "0", - "ccdb-url": "http://alice-ccdb.cern.ch", - "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "true", - "networkPathCCDB": "Analysis/PID/TPC/ML", - "networkPathLocally": "network.onnx", - "networkSetNumThreads": "0", - "param-file": "", - "pid-al": "-1", - "pid-de": "-1", - "pid-el": "-1", - "pid-he": "-1", - "pid-ka": "-1", - "pid-mu": "-1", - "pid-pi": "-1", - "pid-pr": "-1", - "pid-tr": "-1", - "recoPass": "", - "skipTPCOnly": "false", - "useNetworkCorrection": "false" + "trackSelection": "1" }, "track-extension": { "processRun2": "false", @@ -504,4 +514,4 @@ "ptMax": "1e+10", "ptMin": "0.1" } -} \ No newline at end of file +} diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json index deeee5c89..c97f206f3 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json @@ -1,4 +1,50 @@ { + "EMCClusterMonitorTask": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "0", + "processCollisions": "1", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "1", + "processCollisions": "0", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, "McCollisions": {}, "McParticles": {}, "McTrackLabels": {}, @@ -14,12 +60,77 @@ "processRun2": "false", "processRun3": "true" }, + "cell-monitor": { + "maxCellTimeMain": "100", + "minCellAmplitude": "0", + "minCellAmplitudeTimeHists": "0.3", + "minCellTimeMain": "-50", + "selectBCID": "all", + "vetoBCID": "" + }, "check-mc-particles-indices": { "debugMode": "0" }, "check-mc-particles-indices-grouped": { "debugMode": "1" }, + "emc-tmmonitor": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "hasPropagatedTracks": "1", + "maxTime": "20", + "minM02": "0.1", + "minTime": "-25", + "processCollisions": "1", + "selectBCID": "all", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + }, + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "usePionRejection": "0", + "vertexCut": "-1", + "vetoBCID": "" + }, + "emcal-correction-task": { + "clusterDefinition": "kV3Default", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellFraction": "0.97", + "exoticCellInCrossMinAmplitude": "0.1", + "exoticCellMinAmplitude": "4", + "hasPropagatedTracks": "0", + "hasShaperCorrection": "1", + "isMC": "1", + "logWeight": "4.5", + "maxMatchingDistance": "0.4", + "nonlinearityFunction": "MC_TestbeamFinal", + "processFull": "0", + "processMCFull": "1", + "processStandalone": "0", + "selectedCellType": "1", + "useWeightExotic": "0" + }, "event-selection-qa-task": { "isMC": "true", "processMCRun3": "true", @@ -952,6 +1063,10 @@ "v0setting_dcav0dau": "1", "v0setting_radius": "0.9" }, + "pid-multiplicity": { + "processIU": "0", + "processStandard": "1" + }, "pseudorapidity-density": { "estimatorEta": "1", "exclusionPhi": { @@ -1057,8 +1172,18 @@ "processMuonOnlyWithCov": "false", "processOnlyBCs": "true" }, - "tof-signal": "", - "tpc-pid-qa": { + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "1", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", "etaBins": { "values": [ 50, @@ -1066,43 +1191,52 @@ 1 ] }, - "dEdxBins": { + "expSigmaBins": { "values": [ - 3000, + 200, 0, - 1000 + 200 + ] + }, + "logAxis": "0", + "maxP": "5", + "minP": "0.1", + "nBinsP": "400", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 ] }, - "applyRapidityCut": "0", - "processFullWithTOFDeuteron": "0", "phiBins": { "values": [ - 50, + 25, 0, 6.283185307179586 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "1", - "processKaon": "0", "processAlpha": "0", - "processFullWithTOFPion": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "0", "processFullHelium3": "1", - "minP": "0.01", - "enableDeDxPlot": "1", - "processFullWithTOFElectron": "0", - "logAxis": "1", - "processFullWithTOFAlpha": "0", "processFullKaon": "1", + "processFullMuon": "0", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processHelium3": "0", + "processKaon": "0", "processMuon": "0", - "processFullDeuteron": "1", - "processFullWithTOFMuon": "0", - "processFullWithTOFProton": "0", - "maxP": "5", - "processFullAlpha": "0", - "processElectron": "0", - "processDeuteron": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "produceDeltaTEtaPhiMap": "0", + "ptDeltaTEtaPhiMap": "3", + "splitSignalPerCharge": "1", "trackLengthBins": { "values": [ 50, @@ -1110,33 +1244,40 @@ 1000 ] }, - "processFullProton": "1", - "processFullWithTOFKaon": "0", - "minTPCNcls": "0", - "processFullMuon": "1", - "processFullWithTOFTriton": "0", - "nSigmaBins": { - "values": [ - 400, - -10, - 10 - ] - }, - "trackSelection": "1", - "processFullWithTOFHelium3": "0", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "trackSelection": "1" + }, + "tof-signal": "", + "tpc-pid-full": { + "autofetchNetworks": "1", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "1", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "0" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "0", + "dEdxBins": { "values": [ - 100, + 3000, 0, - 200 + 1000 ] }, - "applyEvSel": "2", - "nBinsP": "400", "deltaBins": { "values": [ 100, @@ -1144,9 +1285,7 @@ 1000 ] }, - "processFullPion": "1" - }, - "tof-pid-qa": { + "enableDeDxPlot": "1", "etaBins": { "values": [ 50, @@ -1154,42 +1293,18 @@ 1 ] }, - "applyRapidityCut": "1", - "phiBins": { + "expSigmaBins": { "values": [ - 25, + 100, 0, - 6.283185307179586 + 200 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "0", - "processKaon": "0", - "processAlpha": "0", - "processFullHelium3": "1", - "minP": "0.1", - "logAxis": "0", - "processFullKaon": "1", - "produceDeltaTEtaPhiMap": "0", - "processMuon": "0", - "processFullDeuteron": "1", + "logAxis": "1", "maxP": "5", - "processFullAlpha": "0", - "ptDeltaTEtaPhiMap": "3", - "processElectron": "0", - "processDeuteron": "0", - "trackLengthBins": { - "values": [ - 50, - 0, - 1000 - ] - }, - "processFullProton": "1", - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", - "processFullMuon": "0", + "minP": "0.01", + "minTPCNcls": "0", + "nBinsP": "400", "nSigmaBins": { "values": [ 400, @@ -1197,53 +1312,49 @@ 10 ] }, - "trackSelection": "1", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "phiBins": { "values": [ - 200, + 50, 0, - 200 + 6.283185307179586 ] }, - "applyEvSel": "2", - "nBinsP": "400", - "deltaBins": { + "processAlpha": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "1", + "processFullHelium3": "1", + "processFullKaon": "1", + "processFullMuon": "1", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processFullWithTOFAlpha": "0", + "processFullWithTOFDeuteron": "0", + "processFullWithTOFElectron": "0", + "processFullWithTOFHelium3": "0", + "processFullWithTOFKaon": "0", + "processFullWithTOFMuon": "0", + "processFullWithTOFPion": "0", + "processFullWithTOFProton": "0", + "processFullWithTOFTriton": "0", + "processHelium3": "0", + "processKaon": "0", + "processMuon": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "splitSignalPerCharge": "1", + "trackLengthBins": { "values": [ - 100, - -1000, + 50, + 0, 1000 ] }, - "processFullPion": "1" - }, - "tpc-pid-full": { - "autofetchNetworks": "1", - "ccdb-timestamp": "0", - "ccdb-url": "http://alice-ccdb.cern.ch", - "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "1", - "networkPathCCDB": "Analysis/PID/TPC/ML", - "networkPathLocally": "network.onnx", - "networkSetNumThreads": "0", - "param-file": "", - "pid-al": "-1", - "pid-de": "-1", - "pid-el": "-1", - "pid-he": "-1", - "pid-ka": "-1", - "pid-mu": "-1", - "pid-pi": "-1", - "pid-pr": "-1", - "pid-tr": "-1", - "useNetworkCorrection": "0" - }, - "pid-multiplicity": { - "processIU": "0", - "processStandard": "1" + "trackSelection": "1" }, "track-extension": { "processRun2": "false", @@ -1267,116 +1378,5 @@ "produceFBextendedTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" - }, - "emcal-correction-task": { - "nonlinearityFunction": "MC_TestbeamFinal", - "logWeight": "4.5", - "exoticCellMinAmplitude": "4", - "clusterDefinition": "kV3Default", - "useWeightExotic": "0", - "disableNonLin": "0", - "exoticCellDiffTime": "1e+06", - "exoticCellInCrossMinAmplitude": "0.1", - "maxMatchingDistance": "0.4", - "exoticCellFraction": "0.97", - "hasShaperCorrection": "1", - "isMC": "1", - "hasPropagatedTracks": "0", - "processFull": "0", - "selectedCellType": "1", - "processStandalone": "0", - "processMCFull": "1" - }, - "cell-monitor": { - "minCellTimeMain": "-50", - "maxCellTimeMain": "100", - "minCellAmplitudeTimeHists": "0.3", - "vetoBCID": "", - "selectBCID": "all", - "minCellAmplitude": "0" - }, - "EMCClusterMonitorTaskAmbiguous": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "0", - "doEventSel": "0", - "processAmbiguous": "1", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "EMCClusterMonitorTask": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "1", - "doEventSel": "0", - "processAmbiguous": "0", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "emc-tmmonitor": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "doEventSel": "0", - "maxTime": "20", - "clusterDefinition": "10", - "minM02": "0.1", - "vetoBCID": "", - "processCollisions": "1", - "hasPropagatedTracks": "1", - "usePionRejection": "0", - "minTime": "-25", - "tpcNsigmaPion": { - "values": [ - -3, - 3 - ] - }, - "selectBCID": "all", - "tpcNsigmaElectron": { - "values": [ - -1, - 3 - ] - }, - "vertexCut": "-1", - "tpcNsigmaBack": { - "values": [ - -10, - -4 - ] - } } } diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json index 58f8cf022..c754b963c 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -292,8 +292,18 @@ "rapidity": "0.5", "v0cospa": "0.995" }, - "tof-signal": "", - "tpc-pid-qa": { + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "1", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", "etaBins": { "values": [ 50, @@ -301,43 +311,52 @@ 1 ] }, - "dEdxBins": { + "expSigmaBins": { "values": [ - 3000, + 200, 0, - 1000 + 200 + ] + }, + "logAxis": "0", + "maxP": "5", + "minP": "0.1", + "nBinsP": "400", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 ] }, - "applyRapidityCut": "0", - "processFullWithTOFDeuteron": "0", "phiBins": { "values": [ - 50, + 25, 0, 6.283185307179586 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "1", - "processKaon": "0", "processAlpha": "0", - "processFullWithTOFPion": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "0", "processFullHelium3": "1", - "minP": "0.01", - "enableDeDxPlot": "1", - "processFullWithTOFElectron": "0", - "logAxis": "1", - "processFullWithTOFAlpha": "0", "processFullKaon": "1", + "processFullMuon": "0", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processHelium3": "0", + "processKaon": "0", "processMuon": "0", - "processFullDeuteron": "1", - "processFullWithTOFMuon": "0", - "processFullWithTOFProton": "0", - "maxP": "5", - "processFullAlpha": "0", - "processElectron": "0", - "processDeuteron": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "produceDeltaTEtaPhiMap": "0", + "ptDeltaTEtaPhiMap": "3", + "splitSignalPerCharge": "1", "trackLengthBins": { "values": [ 50, @@ -345,33 +364,42 @@ 1000 ] }, - "processFullProton": "1", - "processFullWithTOFKaon": "0", - "minTPCNcls": "0", - "processFullMuon": "1", - "processFullWithTOFTriton": "0", - "nSigmaBins": { - "values": [ - 400, - -10, - 10 - ] - }, - "trackSelection": "1", - "processFullWithTOFHelium3": "0", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "trackSelection": "1" + }, + "tof-signal": "", + "tpc-pid-full": { + "autofetchNetworks": "true", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "true", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "recoPass": "", + "skipTPCOnly": "false", + "useNetworkCorrection": "false" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "0", + "dEdxBins": { "values": [ - 100, + 3000, 0, - 200 + 1000 ] }, - "applyEvSel": "2", - "nBinsP": "400", "deltaBins": { "values": [ 100, @@ -379,9 +407,7 @@ 1000 ] }, - "processFullPion": "1" - }, - "tof-pid-qa": { + "enableDeDxPlot": "1", "etaBins": { "values": [ 50, @@ -389,42 +415,18 @@ 1 ] }, - "applyRapidityCut": "1", - "phiBins": { + "expSigmaBins": { "values": [ - 25, + 100, 0, - 6.283185307179586 + 200 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "0", - "processKaon": "0", - "processAlpha": "0", - "processFullHelium3": "1", - "minP": "0.1", - "logAxis": "0", - "processFullKaon": "1", - "produceDeltaTEtaPhiMap": "0", - "processMuon": "0", - "processFullDeuteron": "1", + "logAxis": "1", "maxP": "5", - "processFullAlpha": "0", - "ptDeltaTEtaPhiMap": "3", - "processElectron": "0", - "processDeuteron": "0", - "trackLengthBins": { - "values": [ - 50, - 0, - 1000 - ] - }, - "processFullProton": "1", - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", - "processFullMuon": "0", + "minP": "0.01", + "minTPCNcls": "0", + "nBinsP": "400", "nSigmaBins": { "values": [ 400, @@ -432,51 +434,49 @@ 10 ] }, - "trackSelection": "1", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "phiBins": { "values": [ - 200, + 50, 0, - 200 + 6.283185307179586 ] }, - "applyEvSel": "2", - "nBinsP": "400", - "deltaBins": { + "processAlpha": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "1", + "processFullHelium3": "1", + "processFullKaon": "1", + "processFullMuon": "1", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processFullWithTOFAlpha": "0", + "processFullWithTOFDeuteron": "0", + "processFullWithTOFElectron": "0", + "processFullWithTOFHelium3": "0", + "processFullWithTOFKaon": "0", + "processFullWithTOFMuon": "0", + "processFullWithTOFPion": "0", + "processFullWithTOFProton": "0", + "processFullWithTOFTriton": "0", + "processHelium3": "0", + "processKaon": "0", + "processMuon": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "splitSignalPerCharge": "1", + "trackLengthBins": { "values": [ - 100, - -1000, + 50, + 0, 1000 ] }, - "processFullPion": "1" - }, - "tpc-pid-full": { - "autofetchNetworks": "true", - "ccdb-timestamp": "0", - "ccdb-url": "http://alice-ccdb.cern.ch", - "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "true", - "networkPathCCDB": "Analysis/PID/TPC/ML", - "networkPathLocally": "network.onnx", - "networkSetNumThreads": "0", - "param-file": "", - "pid-al": "-1", - "pid-de": "-1", - "pid-el": "-1", - "pid-he": "-1", - "pid-ka": "-1", - "pid-mu": "-1", - "pid-pi": "-1", - "pid-pr": "-1", - "pid-tr": "-1", - "recoPass": "", - "skipTPCOnly": "false", - "useNetworkCorrection": "false" + "trackSelection": "1" }, "track-extension": { "processRun2": "false", @@ -503,4 +503,4 @@ "ptMax": "1e+10", "ptMin": "0.1" } -} \ No newline at end of file +} diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json index 6c20c3dc9..74e9247e9 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -1,4 +1,50 @@ { + "EMCClusterMonitorTask": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "0", + "processCollisions": "1", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, + "EMCClusterMonitorTaskAmbiguous": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "numclusters-binning": { + "values": [ + 201, + -0.5, + 200.5 + ] + }, + "processAmbiguous": "1", + "processCollisions": "0", + "selectBCID": "all", + "vertexCut": "-1", + "vetoBCID": "" + }, "McCollisions": {}, "McParticles": {}, "McTrackLabels": {}, @@ -14,12 +60,77 @@ "processRun2": "false", "processRun3": "true" }, + "cell-monitor": { + "maxCellTimeMain": "100", + "minCellAmplitude": "0", + "minCellAmplitudeTimeHists": "0.3", + "minCellTimeMain": "-50", + "selectBCID": "all", + "vetoBCID": "" + }, "check-mc-particles-indices": { "debugMode": "0" }, "check-mc-particles-indices-grouped": { "debugMode": "1" }, + "emc-tmmonitor": { + "clusterDefinition": "10", + "clustertime-binning": { + "values": [ + 1500, + -600, + 900 + ] + }, + "doEventSel": "0", + "hasPropagatedTracks": "1", + "maxTime": "20", + "minM02": "0.1", + "minTime": "-25", + "processCollisions": "1", + "selectBCID": "all", + "tpcNsigmaBack": { + "values": [ + -10, + -4 + ] + }, + "tpcNsigmaElectron": { + "values": [ + -1, + 3 + ] + }, + "tpcNsigmaPion": { + "values": [ + -3, + 3 + ] + }, + "usePionRejection": "0", + "vertexCut": "-1", + "vetoBCID": "" + }, + "emcal-correction-task": { + "clusterDefinition": "kV3Default", + "disableNonLin": "0", + "exoticCellDiffTime": "1e+06", + "exoticCellFraction": "0.97", + "exoticCellInCrossMinAmplitude": "0.1", + "exoticCellMinAmplitude": "4", + "hasPropagatedTracks": "0", + "hasShaperCorrection": "1", + "isMC": "1", + "logWeight": "4.5", + "maxMatchingDistance": "0.4", + "nonlinearityFunction": "MC_TestbeamFinal", + "processFull": "0", + "processMCFull": "1", + "processStandalone": "0", + "selectedCellType": "1", + "useWeightExotic": "0" + }, "event-selection-qa-task": { "isMC": "true", "processMCRun3": "true", @@ -952,6 +1063,10 @@ "v0setting_dcav0dau": "1", "v0setting_radius": "0.9" }, + "pid-multiplicity": { + "processIU": "0", + "processStandard": "1" + }, "pseudorapidity-density": { "estimatorEta": "1", "exclusionPhi": { @@ -1056,8 +1171,18 @@ "processMuonOnlyWithCov": "false", "processOnlyBCs": "true" }, - "tof-signal": "", - "tpc-pid-qa": { + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "1", + "deltaBins": { + "values": [ + 100, + -1000, + 1000 + ] + }, + "enableEvTimeSplitting": "0", + "enableVsMomentumHistograms": "0", "etaBins": { "values": [ 50, @@ -1065,43 +1190,52 @@ 1 ] }, - "dEdxBins": { + "expSigmaBins": { "values": [ - 3000, + 200, 0, - 1000 + 200 + ] + }, + "logAxis": "0", + "maxP": "5", + "minP": "0.1", + "nBinsP": "400", + "nSigmaBins": { + "values": [ + 400, + -10, + 10 ] }, - "applyRapidityCut": "0", - "processFullWithTOFDeuteron": "0", "phiBins": { "values": [ - 50, + 25, 0, 6.283185307179586 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "1", - "processKaon": "0", "processAlpha": "0", - "processFullWithTOFPion": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "0", "processFullHelium3": "1", - "minP": "0.01", - "enableDeDxPlot": "1", - "processFullWithTOFElectron": "0", - "logAxis": "1", - "processFullWithTOFAlpha": "0", "processFullKaon": "1", + "processFullMuon": "0", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processHelium3": "0", + "processKaon": "0", "processMuon": "0", - "processFullDeuteron": "1", - "processFullWithTOFMuon": "0", - "processFullWithTOFProton": "0", - "maxP": "5", - "processFullAlpha": "0", - "processElectron": "0", - "processDeuteron": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "produceDeltaTEtaPhiMap": "0", + "ptDeltaTEtaPhiMap": "3", + "splitSignalPerCharge": "1", "trackLengthBins": { "values": [ 50, @@ -1109,33 +1243,40 @@ 1000 ] }, - "processFullProton": "1", - "processFullWithTOFKaon": "0", - "minTPCNcls": "0", - "processFullMuon": "1", - "processFullWithTOFTriton": "0", - "nSigmaBins": { - "values": [ - 400, - -10, - 10 - ] - }, - "trackSelection": "1", - "processFullWithTOFHelium3": "0", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "trackSelection": "1" + }, + "tof-signal": "", + "tpc-pid-full": { + "autofetchNetworks": "1", + "ccdb-timestamp": "0", + "ccdb-url": "http://alice-ccdb.cern.ch", + "ccdbPath": "Analysis/PID/TPC/Response", + "enableNetworkOptimizations": "1", + "networkPathCCDB": "Analysis/PID/TPC/ML", + "networkPathLocally": "network.onnx", + "networkSetNumThreads": "0", + "param-file": "", + "pid-al": "-1", + "pid-de": "-1", + "pid-el": "-1", + "pid-he": "-1", + "pid-ka": "-1", + "pid-mu": "-1", + "pid-pi": "-1", + "pid-pr": "-1", + "pid-tr": "-1", + "useNetworkCorrection": "0" + }, + "tpc-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "0", + "dEdxBins": { "values": [ - 100, + 3000, 0, - 200 + 1000 ] }, - "applyEvSel": "2", - "nBinsP": "400", "deltaBins": { "values": [ 100, @@ -1143,9 +1284,7 @@ 1000 ] }, - "processFullPion": "1" - }, - "tof-pid-qa": { + "enableDeDxPlot": "1", "etaBins": { "values": [ 50, @@ -1153,42 +1292,18 @@ 1 ] }, - "applyRapidityCut": "1", - "phiBins": { + "expSigmaBins": { "values": [ - 25, + 100, 0, - 6.283185307179586 + 200 ] }, - "processProton": "0", - "processHelium3": "0", - "processFullElectron": "0", - "processKaon": "0", - "processAlpha": "0", - "processFullHelium3": "1", - "minP": "0.1", - "logAxis": "0", - "processFullKaon": "1", - "produceDeltaTEtaPhiMap": "0", - "processMuon": "0", - "processFullDeuteron": "1", + "logAxis": "1", "maxP": "5", - "processFullAlpha": "0", - "ptDeltaTEtaPhiMap": "3", - "processElectron": "0", - "processDeuteron": "0", - "trackLengthBins": { - "values": [ - 50, - 0, - 1000 - ] - }, - "processFullProton": "1", - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", - "processFullMuon": "0", + "minP": "0.01", + "minTPCNcls": "0", + "nBinsP": "400", "nSigmaBins": { "values": [ 400, @@ -1196,53 +1311,49 @@ 10 ] }, - "trackSelection": "1", - "processFullTriton": "1", - "processTriton": "0", - "processPion": "0", - "splitSignalPerCharge": "1", - "expSigmaBins": { + "phiBins": { "values": [ - 200, + 50, 0, - 200 + 6.283185307179586 ] }, - "applyEvSel": "2", - "nBinsP": "400", - "deltaBins": { + "processAlpha": "0", + "processDeuteron": "0", + "processElectron": "0", + "processFullAlpha": "0", + "processFullDeuteron": "1", + "processFullElectron": "1", + "processFullHelium3": "1", + "processFullKaon": "1", + "processFullMuon": "1", + "processFullPion": "1", + "processFullProton": "1", + "processFullTriton": "1", + "processFullWithTOFAlpha": "0", + "processFullWithTOFDeuteron": "0", + "processFullWithTOFElectron": "0", + "processFullWithTOFHelium3": "0", + "processFullWithTOFKaon": "0", + "processFullWithTOFMuon": "0", + "processFullWithTOFPion": "0", + "processFullWithTOFProton": "0", + "processFullWithTOFTriton": "0", + "processHelium3": "0", + "processKaon": "0", + "processMuon": "0", + "processPion": "0", + "processProton": "0", + "processTriton": "0", + "splitSignalPerCharge": "1", + "trackLengthBins": { "values": [ - 100, - -1000, + 50, + 0, 1000 ] }, - "processFullPion": "1" - }, - "tpc-pid-full": { - "autofetchNetworks": "1", - "ccdb-timestamp": "0", - "ccdb-url": "http://alice-ccdb.cern.ch", - "ccdbPath": "Analysis/PID/TPC/Response", - "enableNetworkOptimizations": "1", - "networkPathCCDB": "Analysis/PID/TPC/ML", - "networkPathLocally": "network.onnx", - "networkSetNumThreads": "0", - "param-file": "", - "pid-al": "-1", - "pid-de": "-1", - "pid-el": "-1", - "pid-he": "-1", - "pid-ka": "-1", - "pid-mu": "-1", - "pid-pi": "-1", - "pid-pr": "-1", - "pid-tr": "-1", - "useNetworkCorrection": "0" - }, - "pid-multiplicity": { - "processIU": "0", - "processStandard": "1" + "trackSelection": "1" }, "track-extension": { "processRun2": "false", @@ -1266,116 +1377,5 @@ "produceFBextendedTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" - }, - "emcal-correction-task": { - "nonlinearityFunction": "MC_TestbeamFinal", - "logWeight": "4.5", - "exoticCellMinAmplitude": "4", - "clusterDefinition": "kV3Default", - "useWeightExotic": "0", - "disableNonLin": "0", - "exoticCellDiffTime": "1e+06", - "exoticCellInCrossMinAmplitude": "0.1", - "maxMatchingDistance": "0.4", - "exoticCellFraction": "0.97", - "hasShaperCorrection": "1", - "isMC": "1", - "hasPropagatedTracks": "0", - "processFull": "0", - "selectedCellType": "1", - "processStandalone": "0", - "processMCFull": "1" - }, - "cell-monitor": { - "minCellTimeMain": "-50", - "maxCellTimeMain": "100", - "minCellAmplitudeTimeHists": "0.3", - "vetoBCID": "", - "selectBCID": "all", - "minCellAmplitude": "0" - }, - "EMCClusterMonitorTaskAmbiguous": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "0", - "doEventSel": "0", - "processAmbiguous": "1", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "EMCClusterMonitorTask": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "processCollisions": "1", - "doEventSel": "0", - "processAmbiguous": "0", - "clusterDefinition": "10", - "numclusters-binning": { - "values": [ - 201, - -0.5, - 200.5 - ] - }, - "vetoBCID": "", - "selectBCID": "all", - "vertexCut": "-1" - }, - "emc-tmmonitor": { - "clustertime-binning": { - "values": [ - 1500, - -600, - 900 - ] - }, - "doEventSel": "0", - "maxTime": "20", - "clusterDefinition": "10", - "minM02": "0.1", - "vetoBCID": "", - "processCollisions": "1", - "hasPropagatedTracks": "1", - "usePionRejection": "0", - "minTime": "-25", - "tpcNsigmaPion": { - "values": [ - -3, - 3 - ] - }, - "selectBCID": "all", - "tpcNsigmaElectron": { - "values": [ - -1, - 3 - ] - }, - "vertexCut": "-1", - "tpcNsigmaBack": { - "values": [ - -10, - -4 - ] - } } } diff --git a/MC/config/analysis_testing/json/format_json.sh b/MC/config/analysis_testing/json/format_json.sh index aac459674..e3b8b34d5 100755 --- a/MC/config/analysis_testing/json/format_json.sh +++ b/MC/config/analysis_testing/json/format_json.sh @@ -2,7 +2,7 @@ # Script to format the json files -for i in $(ls *.json); do +for i in $(find . | grep "\.json"); do case "$i" in *analyses_config.json*) continue From 5d3c5544a3dd8c1a005dcbf299e8b6fd73653e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Thu, 25 Jan 2024 11:35:24 +0100 Subject: [PATCH 0322/1239] LF: add more gap configurations for trigger (#1418) * LF: add more gap configurations for trigger --- .../GeneratorLFStrangenessTriggered_gap2.ini | 12 ++++ .../GeneratorLFStrangenessTriggered_gap3.ini | 12 ++++ .../GeneratorLFStrangenessTriggered_gap5.ini | 12 ++++ .../GeneratorLFStrangenessTriggered_900gev.C | 59 +------------------ .../GeneratorLFStrangenessTriggered_gap2.C | 1 + .../GeneratorLFStrangenessTriggered_gap3.C | 1 + .../GeneratorLFStrangenessTriggered_gap5.C | 1 + 7 files changed, 40 insertions(+), 58 deletions(-) create mode 100644 MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap2.ini create mode 100644 MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap3.ini create mode 100644 MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap5.ini mode change 100644 => 120000 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C create mode 120000 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap2.C create mode 120000 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap3.C create mode 120000 MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap5.C diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap2.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap2.ini new file mode 100644 index 000000000..cfed8beaa --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap2.ini @@ -0,0 +1,12 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 0) +funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 2) + +[GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg +; config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg + +[DecayerPythia8] +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap3.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap3.ini new file mode 100644 index 000000000..a964ff26e --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap3.ini @@ -0,0 +1,12 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 0) +funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 3) + +[GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg +; config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg + +[DecayerPythia8] +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap5.ini b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap5.ini new file mode 100644 index 000000000..67e5f1649 --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFStrangenessTriggered_gap5.ini @@ -0,0 +1,12 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +# funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 0) +funcName=generateLFTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/strangeparticlelist.gun", 5) + +[GeneratorPythia8] # if triggered then this will be used as the background event +# config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg +; config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_136tev.cfg +config=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/pythia8_inel_ropes_136tev.cfg + +[DecayerPythia8] +config[0]=${O2DPG_ROOT}/MC/config/common/pythia8/decayer/base.cfg diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C deleted file mode 100644 index 49ba5d4e2..000000000 --- a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C +++ /dev/null @@ -1,58 +0,0 @@ -int External() -{ - std::string path{"o2sim_Kine.root"}; - int numberOfInjectedSignalsPerEvent{1}; - std::vector injectedPDGs = { - 3334, - -3334, - 3312, - -3312}; - - auto nInjection = injectedPDGs.size(); - - TFile file(path.c_str(), "READ"); - if (file.IsZombie()) { - std::cerr << "Cannot open ROOT file " << path << "\n"; - return 1; - } - - auto tree = (TTree*)file.Get("o2sim"); - if (!tree) { - std::cerr << "Cannot find tree o2sim in file " << path << "\n"; - return 1; - } - std::vector* tracks{}; - tree->SetBranchAddress("MCTrack", &tracks); - - std::vector nSignal; - for (int i = 0; i < nInjection; i++) { - nSignal.push_back(0); - } - - auto nEvents = tree->GetEntries(); - for (int i = 0; i < nEvents; i++) { - auto check = tree->GetEntry(i); - for (int idxMCTrack = 0; idxMCTrack < tracks->size(); ++idxMCTrack) { - auto track = tracks->at(idxMCTrack); - auto pdg = track.GetPdgCode(); - auto it = std::find(injectedPDGs.begin(), injectedPDGs.end(), pdg); - int index = std::distance(injectedPDGs.begin(), it); // index of injected PDG - if (it != injectedPDGs.end()) // found - { - // count signal PDG - nSignal[index]++; - } - } - } - std::cout << "--------------------------------\n"; - std::cout << "# Events: " << nEvents << "\n"; - for (int i = 0; i < nInjection; i++) { - std::cout << "# Injected nuclei \n"; - std::cout << injectedPDGs[i] << ": " << nSignal[i] << "\n"; - if (nSignal[i] == 0) { - std::cerr << "No generated: " << injectedPDGs[i] << "\n"; - return 1; // At least one of the injected particles should be generated - } - } - return 0; -} \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C new file mode 120000 index 000000000..363dcd11d --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_900gev.C @@ -0,0 +1 @@ +GeneratorLFStrangenessTriggered.C \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap2.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap2.C new file mode 120000 index 000000000..363dcd11d --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap2.C @@ -0,0 +1 @@ +GeneratorLFStrangenessTriggered.C \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap3.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap3.C new file mode 120000 index 000000000..363dcd11d --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap3.C @@ -0,0 +1 @@ +GeneratorLFStrangenessTriggered.C \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap5.C b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap5.C new file mode 120000 index 000000000..363dcd11d --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFStrangenessTriggered_gap5.C @@ -0,0 +1 @@ +GeneratorLFStrangenessTriggered.C \ No newline at end of file From 5095aaeee94193020fd51bce2c8ad25fd46405dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Thu, 25 Jan 2024 11:37:06 +0100 Subject: [PATCH 0323/1239] Update hadronic cross section in injected MC (#1419) * Add xsection variation for nuclei --- .../GeneratorLFHyperNucleippGapXSection.ini | 9 +++ .../GeneratorLFHyperNucleippGapXSection.C | 1 + MC/config/PWGLF/xsection/g4config_had_x2.in | 73 +++++++++++++++++++ MC/run/PWGLF/run_XSectionVariation.sh | 33 +++++++++ 4 files changed, 116 insertions(+) create mode 100644 MC/config/PWGLF/ini/GeneratorLFHyperNucleippGapXSection.ini create mode 120000 MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGapXSection.C create mode 100644 MC/config/PWGLF/xsection/g4config_had_x2.in create mode 100644 MC/run/PWGLF/run_XSectionVariation.sh diff --git a/MC/config/PWGLF/ini/GeneratorLFHyperNucleippGapXSection.ini b/MC/config/PWGLF/ini/GeneratorLFHyperNucleippGapXSection.ini new file mode 100644 index 000000000..68f4bd06b --- /dev/null +++ b/MC/config/PWGLF/ini/GeneratorLFHyperNucleippGapXSection.ini @@ -0,0 +1,9 @@ +[GeneratorExternal] +fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C +funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/hypernuclei.gun", 5) + +[GeneratorPythia8] +config=${O2_ROOT}/share/Generators/egconfig/pythia8_inel.cfg + +[G4] +configMacroFile=${O2DPG_ROOT}/MC/config/PWGLF/xsection/g4config_had_x2.in \ No newline at end of file diff --git a/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGapXSection.C b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGapXSection.C new file mode 120000 index 000000000..9bd97e64f --- /dev/null +++ b/MC/config/PWGLF/ini/tests/GeneratorLFHyperNucleippGapXSection.C @@ -0,0 +1 @@ +GeneratorLFHyperNucleippGap.C \ No newline at end of file diff --git a/MC/config/PWGLF/xsection/g4config_had_x2.in b/MC/config/PWGLF/xsection/g4config_had_x2.in new file mode 100644 index 000000000..d1e3dab05 --- /dev/null +++ b/MC/config/PWGLF/xsection/g4config_had_x2.in @@ -0,0 +1,73 @@ +# Geant4 standard configuration based on AliDPG commit 7650a5b + +/control/verbose 2 +/mcVerbose/all 1 +/mcVerbose/geometryManager 1 +/mcVerbose/opGeometryManager 1 +/mcTracking/loopVerbose 1 +/mcVerbose/composedPhysicsList 2 +/mcVerbose/runAction 2 # For looping thresholds control +#/tracking/verbose 1 +#//control/cout/ignoreThreadsExcept 0 + +/mcPhysics/rangeCuts 0.001 mm +/mcRegions/setRangePrecision 5 +/mcTracking/skipNeutrino true +/mcDet/setIsMaxStepInLowDensityMaterials true +/mcDet/setMaxStepInLowDensityMaterials 10 m +/mcMagField/setConstDistance 1 mm +/mcDet/setIsZeroMagField true +/mcControl/useRootRandom true # couple G4 random seed to gRandom + +# optical + +/process/optical/verbose 0 +/process/optical/processActivation Scintillation 0 +/process/optical/processActivation OpWLS 0 +/process/optical/processActivation OpMieHG 0 +/process/optical/cerenkov/setTrackSecondariesFirst false +/mcMagField/stepperType NystromRK4 + +# PAI for TRD +# Geant4 VMC >= v3.2 +/mcPhysics/emModel/setEmModel PAI +/mcPhysics/emModel/setRegions TRD_Gas-mix +/mcPhysics/emModel/setParticles all +/mcPrimaryGenerator/skipUnknownParticles true # don't crash when seeing unknown ion etc. (issue warning) + +# +# Precise Msc for EMCAL +# +# Geant4 VMC >= v3.2 +/mcPhysics/emModel/setEmModel SpecialUrbanMsc +/mcPhysics/emModel/setRegions EMC_Lead$ EMC_Scintillator$ +/mcPhysics/emModel/setParticles e- e+ + +# +# Adding extra lines for fixing tracking bias +# +/mcMagField/setDeltaIntersection 1.0e-05 mm +/mcMagField/setMinimumEpsilonStep 0.5e-05 +/mcMagField/setMaximumEpsilonStep 1.0e-05 +/mcMagField/printParameters + +# Change default parameters for killing looping particles +# +/mcPhysics/useHighLooperThresholds +/mcRun/setLooperThresholdImportantEnergy 100. MeV + +# Define media with the INCLXX physics list; here basically in all ITS media +#/mcVerbose/biasingConfigurationManager 3 +/mcPhysics/biasing/setModel inclxx +/mcPhysics/biasing/setRegions ITS_AIR$ ITS_WATER$ ITS_COPPER$ ITS_KAPTON(POLYCH2)$ ITS_GLUE_IBFPC$ ITS_CERAMIC$ ITS_K13D2U2k$ ITS_K13D2U120$ ITS_F6151B05M$ ITS_M60J3K$ ITS_M55J6K$ ITS_FGS003$ ITS_CarbonFleece$ ITS_PEEKCF30$ ITS_GLUE$ ITS_ALUMINUM$ ITS_INOX304$ ALPIDE_METALSTACK$ ALPIDE_SI$ +/mcPhysics/biasing/setParticles proton neutron pi+ pi- + + +# Scale hadronic cross section +/mcPhysics/setCrossSectionFactor deuteron hadElastic 2. +/mcPhysics/setCrossSectionFactor anti_deuteron hadElastic 2. +/mcPhysics/setCrossSectionFactor triton hadElastic 2. +/mcPhysics/setCrossSectionFactor anti_triton hadElastic 2. +/mcPhysics/setCrossSectionFactor he3 hadElastic 2. +/mcPhysics/setCrossSectionFactor anti_he3 hadElastic 2. + diff --git a/MC/run/PWGLF/run_XSectionVariation.sh b/MC/run/PWGLF/run_XSectionVariation.sh new file mode 100644 index 000000000..027f6d0a0 --- /dev/null +++ b/MC/run/PWGLF/run_XSectionVariation.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# +# A example workflow MC->RECO->AOD for a simple pp min bias +# production, targetting test beam conditions. + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +# ----------- START ACTUAL JOB ----------------------------- + +NWORKERS=${NWORKERS:-8} +MODULES="--skipModules ZDC" +SIMENGINE=${SIMENGINE:-TGeant3} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NTIMEFRAMES=${NTIMEFRAMES:-1} +INTRATE=${INTRATE:-500000} +SYSTEM=${SYSTEM:-pp} +ENERGY=${ENERGY:-13600} +[[ ${SPLITID} != "" ]] && SEED="-seed ${SPLITID}" || SEED="" + +# create workflow +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${ENERGY} -col ${SYSTEM} -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} -confKey "Diamond.width[0]=0.1;Diamond.width[1]=0.1;Diamond.width[2]=6." -e ${SIMENGINE} ${SEED} -mod "--skipModules ZDC" \ + -ini ${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFHyperNuclei${SYSTEM}Gap.ini + +# run workflow +# allow increased timeframe parallelism with --cpu-limit 32 +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt aod --cpu-limit 32 From 09b7240ea334bf59f8bf055b2db8384853f316d5 Mon Sep 17 00:00:00 2001 From: noferini <9963644+noferini@users.noreply.github.com> Date: Fri, 26 Jan 2024 12:44:34 +0100 Subject: [PATCH 0324/1239] remove use-fit option in tof matcher (to be added in O2 default) --- DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh | 2 +- DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh | 2 +- .../configurations/2022/LHC22f/apass1/setenv_extra.sh | 2 +- .../configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh | 2 +- .../configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh | 2 +- .../production/configurations/2022/extractCalib/setenv_extra.sh | 2 +- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh b/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh index 45c4885ee..b6c1fdf96 100644 --- a/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh +++ b/DATA/production/configurations/2021/OCT/apass4/setenv_extra.sh @@ -65,7 +65,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh b/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh index 45c4885ee..b6c1fdf96 100644 --- a/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh +++ b/DATA/production/configurations/2021/OCT/apass5/setenv_extra.sh @@ -65,7 +65,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh index ca64edd66..648e9a61b 100644 --- a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh @@ -349,7 +349,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$VDRIFTPARAMOPTION;$TRACKTUNETPCINNER;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh b/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh index c70f06a51..42edfbcf5 100644 --- a/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/MayJunePilotBeam/apass1/setenv_extra.sh @@ -95,7 +95,7 @@ export ITSTPCMATCH="tpcitsMatch.maxVDriftUncertainty=0.2;tpcitsMatch.safeMarginT export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow="TPCGasParam.DriftV=$VDRIFT;$ITSEXTRAERR;$ITSTPCMATCH" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="TPCGasParam.DriftV=$VDRIFT;$ITSEXTRAERR" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh b/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh index b2c79141f..095511c42 100644 --- a/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh +++ b/DATA/production/configurations/2022/MayJunePilotBeam/apass2/setenv_extra.sh @@ -111,7 +111,7 @@ if [[ $WORKFLOW_DETECTORS =~ (^|,)"FT0"(,|$) ]] ; then fi # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$ITSEXTRAERR;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/2022/extractCalib/setenv_extra.sh b/DATA/production/configurations/2022/extractCalib/setenv_extra.sh index a4fac7e44..10bf1543c 100644 --- a/DATA/production/configurations/2022/extractCalib/setenv_extra.sh +++ b/DATA/production/configurations/2022/extractCalib/setenv_extra.sh @@ -69,7 +69,7 @@ if [[ $WORKFLOW_DETECTORS =~ (^|,)"FT0"(,|$) ]] ; then fi # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="--output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow="$ITSEXTRAERR;$VDRIFTPARAMOPTION;" # ad-hoc settings for TRD matching diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 8409e7935..75b7a63e6 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -452,7 +452,7 @@ export CONFIG_EXTRA_PROCESS_o2_tpcits_match_workflow+=";$ITSEXTRAERR;$ITSTPCMATC has_detector FT0 && export ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow="$ARGS_EXTRA_PROCESS_o2_tpcits_match_workflow --use-ft0" # ad-hoc settings for TOF matching -export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia --use-fit" +export ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow="$ARGS_EXTRA_PROCESS_o2_tof_matcher_workflow --output-type matching-info,calib-info --enable-dia" export CONFIG_EXTRA_PROCESS_o2_tof_matcher_workflow+=";$ITSEXTRAERR;$TRACKTUNETPC;$VDRIFTPARAMOPTION;" if [[ $ALIEN_JDL_LPMPASSNAME == "cpass0" ]]; then From 34e5a29ff00a46ee89b53ec06ce6a8ec40dd2f36 Mon Sep 17 00:00:00 2001 From: shahoian Date: Mon, 29 Jan 2024 15:36:30 +0100 Subject: [PATCH 0325/1239] Enforce RANS_OPT=--ans-version compat for runs<544772 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 75b7a63e6..d9bb33d4b 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -17,6 +17,13 @@ if grep -q /skimmed/ wn.xml ; then export ON_SKIMMED_DATA=1; fi +if [[ $RUNNUMBER -lt 544772 ]]; then + # these runs were using external dictionaries + : ${RANS_OPT:="--ans-version compat"} + export RANS_OPT +fi +echo "RSRUNNUMBER = $RUNNUMBER RANS_OPT = $RANS_OPT" + # detector list if [[ -n $ALIEN_JDL_WORKFLOWDETECTORS ]]; then export WORKFLOW_DETECTORS=$ALIEN_JDL_WORKFLOWDETECTORS From c52b163c1e63ed7639618078b1ad45976beb411c Mon Sep 17 00:00:00 2001 From: Robert Muenzer Date: Fri, 19 Jan 2024 12:06:06 +0100 Subject: [PATCH 0326/1239] Make nlanes in krypton workflow setable and put default to 36 --- DATA/testing/detectors/TPC/tpc-krypton.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index 4d987d602..f7e871611 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -17,7 +17,7 @@ PROXY_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" CALIB_INSPEC="A:TPC/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" -NLANES=1 +NLANES=36 SESSION="default" ARGS_FILES="keyval.output_dir=/dev/null" @@ -32,6 +32,12 @@ if [[ ! -z ${TPC_KRYPTON_NO_WRITEOUT:-} ]]; then WRITER_TYPE="--writer-type none" fi + +if [[ ! -z ${TPC_KRYPTON_LANES:-} ]]; then + NLANES=${TPC_KRYPTON_LANES} +fi + + # TODO use add_W function from gen_topo_helper_functions.sh to assemble workflow # as done for example in https://github.com/AliceO2Group/O2DPG/blob/master/DATA/production/calib/its-threshold-processing.sh From 388cc482b4a52db118459a0b611e22839364f9d2 Mon Sep 17 00:00:00 2001 From: Robert Muenzer Date: Fri, 19 Jan 2024 15:02:34 +0100 Subject: [PATCH 0327/1239] Fix QC inclusion in workflow --- DATA/testing/detectors/TPC/tpc-krypton.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index f7e871611..5db7bf581 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -6,7 +6,6 @@ source common/getCommonArgs.sh source common/gen_topo_helper_functions.sh - export GLOBAL_SHMSIZE=$(( 128 << 30 )) # GB for the global SHMEM # for kr cluster finder if [ $NUMAGPUIDS != 0 ]; then @@ -22,8 +21,8 @@ SESSION="default" ARGS_FILES="keyval.output_dir=/dev/null" -#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" -QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-qcmn" +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-qcmn" +#QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-qcmn" WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR" @@ -48,7 +47,7 @@ WORKFLOW= add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pipeline tpc-raw-to-digits-0:20 " "\"${ARGS_FILES}\";TPCDigitDump.LastTimeBin=14256" add_W o2-tpc-krypton-clusterer "${WRITER_TYPE} --lanes $NLANES --configFile=\"/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini\"" "\"${ARGS_FILES}\"" -add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" +add_W o2-qc "--config $QC_CONFIG --local --host localhost" WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then @@ -60,7 +59,7 @@ else eval $WORKFLOW fi -##o2-dpl-raw-proxy $ARGS_ALL \ +#o2-dpl-raw-proxy $ARGS_ALL \ # --dataspec "$PROXY_INSPEC" --inject-missing-data \ # --readout-proxy "--channel-config 'name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1'" \ # | o2-tpc-raw-to-digits-workflow $ARGS_ALL \ @@ -73,6 +72,6 @@ fi # ${WRITER_TYPE} \ # --lanes $NLANES \ # --configKeyValues "$ARGS_FILES" \ -## --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ +# --configFile="/home/wiechula/processData/inputFilesTracking/krypton/krBoxCluster.largeBox.cuts.krMap.ini" \ # | o2-qc $ARGS_ALL --config $QC_CONFIG --local --host localhost \ # | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} ${GLOBALDPLOPT} From 1b9da5ea32795351deea1bd673b94f1db6c8a312 Mon Sep 17 00:00:00 2001 From: Robert Muenzer Date: Fri, 26 Jan 2024 08:57:21 +0100 Subject: [PATCH 0328/1239] add setable LANES to krypton raw workflow --- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index f62b09d57..a502ebf7e 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -15,8 +15,14 @@ SESSION="default" ARGS_FILES="keyval.output_dir=/dev/null" HOST=localhost -#QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" -QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" + + +if [[ ! -z ${TPC_KRYPTON_LANES:-} ]]; then + NLANES=${TPC_KRYPTON_LANES} +fi + +QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" +#QC_CONFIG="/o2/components/qc/ANY/any/tpc-krypton-raw-qcmn" @@ -35,7 +41,7 @@ WORKFLOW= add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 add_W o2-tpc-raw-to-digits-workflow "--ignore-grp --input-spec \"$CALIB_INSPEC\" --remove-duplicates --pedestal-url \"http://o2-ccdb.internal\" --pipeline tpc-raw-to-digits-0:24 " "\"${ARGS_FILES}\";TPCDigitDump.LastTimeBin=446" add_W o2-tpc-krypton-raw-filter "${WRITER_TYPE} --lanes $NLANES --threshold-max 20 --time-bins-before 20" "\"${ARGS_FILES}\"" -add_QC_from_consul "${QC_CONFIG}" "--local --host lcoalhost" +add_W o2-qc "--config $QC_CONFIG --local --host localhost" WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" if [ $WORKFLOWMODE == "print" ]; then From acd6d720a1bb6d0f436d99d970a604b1ef5ef164 Mon Sep 17 00:00:00 2001 From: Robert Muenzer Date: Tue, 30 Jan 2024 22:15:16 +0100 Subject: [PATCH 0329/1239] Changed the parameter calling for Writeout disabling --- DATA/testing/detectors/TPC/tpc-krypton-raw.sh | 2 +- DATA/testing/detectors/TPC/tpc-krypton.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh index a502ebf7e..dd92226c4 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton-raw.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton-raw.sh @@ -28,7 +28,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR --max-tf-per-file 8000" -if [[ ! -z ${TPC_KRYPTON_NO_WRITEOUT:-} ]]; then +if [[ "${TPC_KRYPTON_NO_WRITEOUT:-}" == "1" ]]; then WRITER_TYPE="--writer-type none" fi diff --git a/DATA/testing/detectors/TPC/tpc-krypton.sh b/DATA/testing/detectors/TPC/tpc-krypton.sh index 5db7bf581..f7649a0fd 100755 --- a/DATA/testing/detectors/TPC/tpc-krypton.sh +++ b/DATA/testing/detectors/TPC/tpc-krypton.sh @@ -27,7 +27,7 @@ QC_CONFIG="consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/an WRITER_TYPE="--writer-type EPN --meta-output-dir $EPN2EOS_METAFILES_DIR --output-dir $CALIB_DIR" -if [[ ! -z ${TPC_KRYPTON_NO_WRITEOUT:-} ]]; then +if [[ "${TPC_KRYPTON_NO_WRITEOUT:-}" == "1" ]]; then WRITER_TYPE="--writer-type none" fi From 224e43dd6d5c34b739476fca4bc0c1f856209d7c Mon Sep 17 00:00:00 2001 From: Andreas Molander Date: Wed, 31 Jan 2024 13:34:56 +0200 Subject: [PATCH 0330/1239] Add FT0 QA to analysis QC (#1423) --- .../analysis_testing/json/analyses_config.json | 14 ++++++++++++++ .../json/default/pbpb/analysis-testing-data.json | 3 +++ .../json/default/pbpb/analysis-testing-mc.json | 3 +++ .../json/default/pp/analysis-testing-data.json | 3 +++ .../json/default/pp/analysis-testing-mc.json | 3 +++ 5 files changed, 26 insertions(+) diff --git a/MC/config/analysis_testing/json/analyses_config.json b/MC/config/analysis_testing/json/analyses_config.json index e2721c661..63c67b285 100644 --- a/MC/config/analysis_testing/json/analyses_config.json +++ b/MC/config/analysis_testing/json/analyses_config.json @@ -245,6 +245,20 @@ "o2-analysis-pid-tpc-full", "o2-analysis-pid-tpc-base" ] + }, + { + "name": "FT0QA", + "enabled": true, + "expected_output": ["AnalysisResults.root"], + "valid_mc": true, + "valid_data": true, + "tasks": ["o2-analysis-timestamp", + "o2-analysis-track-propagation", + "o2-analysis-trackselection", + "o2-analysis-event-selection", + "o2-analysis-multiplicity-table", + "o2-analysis-ft0-corrected-table", + "o2-analysis-ft0-qa"] } ] } diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json index 69a6cd245..3b8c49112 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json @@ -513,5 +513,8 @@ "produceTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" + }, + "ft0-qa": { + "isLowFlux": "false" } } diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json index c97f206f3..33d4017a6 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-mc.json @@ -1378,5 +1378,8 @@ "produceFBextendedTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" + }, + "ft0-qa": { + "isLowFlux": "false" } } diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json index c754b963c..817d56da9 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -502,5 +502,8 @@ "produceTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" + }, + "ft0-qa": { + "isLowFlux": "true" } } diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json index 74e9247e9..4ab7676e0 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -1377,5 +1377,8 @@ "produceFBextendedTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" + }, + "ft0-qa": { + "isLowFlux": "true" } } From eb3591632fe75ba65ff68353984839c22826a89c Mon Sep 17 00:00:00 2001 From: catalinristea Date: Wed, 31 Jan 2024 17:22:39 +0200 Subject: [PATCH 0331/1239] Update anchorMC.sh - removing the use of ideal MFT alignment (#1420) * Update anchorMC.sh - removing the use of ideal MFT alignment * Update anchorMC.sh - removed CCDB prefetching --- MC/run/ANCHOR/anchorMC.sh | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 0892bf906..d0beecc0a 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -212,21 +212,8 @@ fi TIMESTAMP=`grep "Determined timestamp to be" timestampsampling_${ALIEN_JDL_LPMRUNNUMBER}.log | awk '//{print $6}'` echo "TIMESTAMP IS ${TIMESTAMP}" -# -- PREFETCH CCDB OBJECTS TO DISC -- -# (make sure the right objects at the right timestamp are fetched -# until https://alice.its.cern.ch/jira/browse/O2-2852 is fixed) -# NOTE: In fact, looking at the ticket, it should be fixed now. However, not changing at the moment as further tests would be needed to confirm here. - -CCDBOBJECTS="/CTP/Calib/OrbitReset /GLO/Config/GRPMagField/ /GLO/Config/GRPLHCIF /ITS/Calib/DeadMap /ITS/Calib/NoiseMap /ITS/Calib/ClusterDictionary /TPC/Calib/PadGainFull /TPC/Calib/TopologyGain /TPC/Calib/TimeGain /TPC/Calib/PadGainResidual /TPC/Config/FEEPad /TOF/Calib/Diagnostic /TOF/Calib/LHCphase /TOF/Calib/FEELIGHT /TOF/Calib/ChannelCalib /MFT/Calib/DeadMap /MFT/Calib/NoiseMap /MFT/Calib/ClusterDictionary /FV0/Calibration/ChannelTimeOffset" - -${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS} -d ${ALICEO2_CCDB_LOCALCACHE} --timestamp ${TIMESTAMP} -if [ ! "$?" == "0" ]; then - echo "Problem during CCDB prefetching of ${CCDBOBJECTS}. Exiting." - exit 1 -fi - -# -- Create aligned geometry using ITS and MFT ideal alignments to avoid overlaps in geant -CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align MFT/Calib/Align" +# -- Create aligned geometry using ITS ideal alignment to avoid overlaps in geant +CCDBOBJECTS_IDEAL_MC="ITS/Calib/Align" TIMESTAMP_IDEAL_MC=1 ${O2_ROOT}/bin/o2-ccdb-downloadccdbfile --host http://alice-ccdb.cern.ch/ -p ${CCDBOBJECTS_IDEAL_MC} -d ${ALICEO2_CCDB_LOCALCACHE} --timestamp ${TIMESTAMP_IDEAL_MC} if [ ! "$?" == "0" ]; then @@ -234,7 +221,7 @@ if [ ! "$?" == "0" ]; then exit 1 fi -echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align,MFT/Calib/Align -b +echo "run with echo in pipe" | ${O2_ROOT}/bin/o2-create-aligned-geometry-workflow --configKeyValues "HBFUtils.startTime=${TIMESTAMP}" --condition-remap=file://${ALICEO2_CCDB_LOCALCACHE}=ITS/Calib/Align -b mkdir -p $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned ln -s -f $PWD/o2sim_geometry-aligned.root $ALICEO2_CCDB_LOCALCACHE/GLO/Config/GeometryAligned/snapshot.root From f7aa7fc4a70889c8f4c177bee72ff8886469bc73 Mon Sep 17 00:00:00 2001 From: swenzel Date: Wed, 31 Jan 2024 15:57:31 +0100 Subject: [PATCH 0332/1239] Possibility to take external config for Pythia8 In case of generator pythia8, we have so far always constructed a Pythia8 config file from the parameters given to o2dpg_sim_workflow.py However, some expert users may want to use an external configuration for Pythia8. This commit provides the possibility to do so via sensitivity to the `GeneratorPythia8.config` ConfigurableParam (so far ignored). An example is: ``` ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 14000 -col pp -gen pythia8 -proc cdiff -tf 2 \ -ns 20 -e ${SIMENGINE} \ -j ${NWORKERS} -interactionRate 500000 \ -run 302000 -seed 624 \ -confKey "GeneratorPythia8.config=/SOMEPATH/pythia8_powheg.cfg" ``` The new feature allows expert studies with specially setup Pythia8 configs. The development was motivated from https://its.cern.ch/jira/browse/O2-4549 However, note that options `-proc` `-eCM` etc. might have no effect or are ignored in such cases. --- MC/bin/o2dpg_sim_config.py | 2 ++ MC/bin/o2dpg_sim_workflow.py | 48 +++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/MC/bin/o2dpg_sim_config.py b/MC/bin/o2dpg_sim_config.py index d065998e8..22017086d 100755 --- a/MC/bin/o2dpg_sim_config.py +++ b/MC/bin/o2dpg_sim_config.py @@ -97,6 +97,8 @@ def create_geant_config(args, externalConfigString): # creates generic transport simulation config key values # based on arguments args (run number, energy, ...) originally passed # to o2dpg_sim_workflow.py + # + # returns a dictionary of mainkey -> dictionary of subkey : values config = {} def add(cfg, flatconfig): for entry in flatconfig: diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 19a2e9b9a..4fe8781ee 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -21,7 +21,7 @@ import importlib.util import argparse from os import environ, mkdir, getcwd -from os.path import join, dirname, isdir +from os.path import join, dirname, isdir, isabs import random import json import itertools @@ -505,6 +505,9 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): workflow['stages'].append(TPC_SPACECHARGE_DOWNLOADER_TASK) +# query initial configKey args for signal transport; mainly used to setup generators +simInitialConfigKeys = create_geant_config(args, args.confKey) + # loop over timeframes for tf in range(1, NTIMEFRAMES + 1): TFSEED = SIMSEED + tf @@ -627,19 +630,30 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): SGN_CONFIG_task=createTask(name='gensgnconf_'+str(tf), tf=tf, cwd=timeframeworkdir) SGN_CONFIG_task['cmd'] = 'echo "placeholder / dummy task"' if GENERATOR == 'pythia8': - SGN_CONFIG_task['cmd'] = '${O2DPG_ROOT}/MC/config/common/pythia8/utils/mkpy8cfg.py \ - --output=pythia8.cfg \ - --seed='+str(TFSEED)+' \ - --idA='+str(PDGA)+' \ - --idB='+str(PDGB)+' \ - --eCM='+str(ECMS)+' \ - --eA='+str(EBEAMA)+' \ - --eB='+str(EBEAMB)+' \ - --process='+str(PROCESS)+' \ - --ptHatMin='+str(PTHATMIN)+' \ - --ptHatMax='+str(PTHATMAX) - if WEIGHTPOW > 0: - SGN_CONFIG_task['cmd'] = SGN_CONFIG_task['cmd'] + ' --weightPow=' + str(WEIGHTPOW) + # see if config is given externally + externalPythia8Config = simInitialConfigKeys.get("GeneratorPythia8", {}).get("config", None) + if externalPythia8Config != None: + # check if this refers to a file with ABSOLUTE path + if not isabs(externalPythia8Config): + print ('Error: Argument to GeneratorPythia8.config must be absolute path') + exit (1) + # in this case, we copy the external config to the local dir (maybe not even necessary) + SGN_CONFIG_task['cmd'] = 'cp ' + externalPythia8Config + ' pythia8.cfg' + else: + SGN_CONFIG_task['cmd'] = '${O2DPG_ROOT}/MC/config/common/pythia8/utils/mkpy8cfg.py \ + --output=pythia8.cfg \ + --seed='+str(TFSEED)+' \ + --idA='+str(PDGA)+' \ + --idB='+str(PDGB)+' \ + --eCM='+str(ECMS)+' \ + --eA='+str(EBEAMA)+' \ + --eB='+str(EBEAMB)+' \ + --process='+str(PROCESS)+' \ + --ptHatMin='+str(PTHATMIN)+' \ + --ptHatMax='+str(PTHATMAX) + if WEIGHTPOW > 0: + SGN_CONFIG_task['cmd'] = SGN_CONFIG_task['cmd'] + ' --weightPow=' + str(WEIGHTPOW) + # if we configure pythia8 here --> we also need to adjust the configuration # TODO: we need a proper config container/manager so as to combine these local configs with external configs etc. args.confKey = args.confKey + ";GeneratorPythia8.config=pythia8.cfg" @@ -647,9 +661,11 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): # elif GENERATOR == 'extgen': what do we do if generator is not pythia8? # NOTE: Generator setup might be handled in a different file or different files (one per # possible generator) - workflow['stages'].append(SGN_CONFIG_task) + # determine final conf key for signal simulation + CONFKEY = constructConfigKeyArg(create_geant_config(args, args.confKey)) + # ----------------- # transport signals # ----------------- @@ -657,8 +673,6 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): if (args.pregenCollContext == True): signalneeds.append(PreCollContextTask['name']) - # determine final configKey args for signal transport - CONFKEY = constructConfigKeyArg(create_geant_config(args, args.confKey)) # add embedIntoFile only if embeddPattern does contain a '@' embeddinto= "--embedIntoFile ../bkg_MCHeader.root" if (doembedding & ("@" in args.embeddPattern)) else "" From ec4acee8f0d38616b6fa45809661213c9b938acc Mon Sep 17 00:00:00 2001 From: swenzel Date: Sun, 28 Jan 2024 20:40:05 +0100 Subject: [PATCH 0333/1239] Clean possibly leaked CCDB semaphores at workflow start Use new feature of O2 to scan for leaked CCDB semaphores related to CCDB caches and clean them up before workflow execution. To this end, expand the __global_init__ mechanism with a "cmd" (not just environment variables) field. The pipeline runner will execute such init command before workflows start. Solves a problem, where second run/pass of workflow running hangs due to previously leaked semaphores. --- MC/bin/o2_dpg_workflow_runner.py | 37 +++++++++++++++++++++++++++----- MC/bin/o2dpg_sim_workflow.py | 4 +++- MC/bin/o2dpg_workflow_utils.py | 2 +- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index b53e5bf63..44f56303a 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -836,11 +836,12 @@ def __init__(self, workflowfile, args, jmax=100): self.is_productionmode = args.production_mode == True # os.getenv("ALIEN_PROC_ID") != None self.workflowfile = workflowfile self.workflowspec = load_json(workflowfile) - self.globalenv = self.extract_global_environment(self.workflowspec) # initialize global environment settings - for e in self.globalenv: + self.globalinit = self.extract_global_environment(self.workflowspec) # initialize global environment settings + for e in self.globalinit['env']: if os.environ.get(e, None) == None: - actionlogger.info("Applying global environment from init section " + str(e) + " : " + str(self.globalenv[e])) - os.environ[e] = str(self.globalenv[e]) + value = self.globalinit['env'][e] + actionlogger.info("Applying global environment from init section " + str(e) + " : " + str(value)) + os.environ[e] = str(value) # only keep those tasks that are necessary to be executed based on user's filters self.workflowspec = filter_workflow(self.workflowspec, args.target_tasks, args.target_labels) @@ -936,13 +937,33 @@ def extract_global_environment(self, workflowspec): """ init_index = 0 # this has to be the first task in the workflow globalenv = {} + initcmd = None if workflowspec['stages'][init_index]['name'] == '__global_init_task__': env = workflowspec['stages'][init_index].get('env', None) if env != None: globalenv = { e : env[e] for e in env } + cmd = workflowspec['stages'][init_index].get('cmd', None) + if cmd != 'NO-COMMAND': + initcmd = cmd + del workflowspec['stages'][init_index] - return globalenv + return {"env" : globalenv, "cmd" : initcmd } + + def execute_globalinit_cmd(self, cmd): + actionlogger.info("Executing global setup cmd " + str(cmd)) + # perform the global init command (think of cleanup/setup things to be done in any case) + p = subprocess.Popen(['/bin/bash','-c', cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + + # Check if the command was successful (return code 0) + if p.returncode == 0: + actionlogger.info(stdout.decode()) + else: + # this should be an error + actionlogger.error("Error executing global init function") + return False + return True def get_global_task_name(self, name): """ @@ -1579,6 +1600,12 @@ def speedup_ROOT_Init(): self.produce_script(args.produce_script) exit (0) + # execute the user-given global init cmd for this workflow + globalinitcmd = self.globalinit.get("cmd", None) + if globalinitcmd != None: + if not self.execute_globalinit_cmd(globalinitcmd): + exit (1) + if args.rerun_from: reruntaskfound=False for task in self.workflowspec['stages']: diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 4fe8781ee..70dcafb71 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -315,7 +315,9 @@ def extractVertexArgs(configKeyValuesStr, finalDiamondDict): globalenv['ALICEO2_CCDB_LOCALCACHE'] = environ.get('ALICEO2_CCDB_LOCALCACHE') globalenv['IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE'] = '${ALICEO2_CCDB_LOCALCACHE:+"ON"}' -workflow['stages'].append(createGlobalInitTask(globalenv)) +globalinittask = createGlobalInitTask(globalenv) +globalinittask['cmd'] = 'o2-ccdb-cleansemaphores -p ${ALICEO2_CCDB_LOCALCACHE}' +workflow['stages'].append(globalinittask) #### def getDPL_global_options(bigshm=False, ccdbbackend=True): diff --git a/MC/bin/o2dpg_workflow_utils.py b/MC/bin/o2dpg_workflow_utils.py index 304bb8234..a029e8dee 100755 --- a/MC/bin/o2dpg_workflow_utils.py +++ b/MC/bin/o2dpg_workflow_utils.py @@ -114,7 +114,7 @@ def dump_workflow(workflow, filename, meta=None): to_dump = deepcopy(workflow) for s in to_dump: - if s["cmd"] and taskwrapper_string not in s["cmd"]: + if s["cmd"] and s["name"] != '__global_init_task__' and taskwrapper_string not in s["cmd"]: # insert taskwrapper stuff if not there already, only do it if cmd string is not empty s['cmd'] = '. ' + taskwrapper_string + ' ' + s['name']+'.log \'' + s['cmd'] + '\'' # remove unnecessary whitespaces for better readibility From 1e452045c48b0ff077ddc8073853b54a5e00f3c9 Mon Sep 17 00:00:00 2001 From: ddobrigk Date: Mon, 29 Jan 2024 19:51:12 +0100 Subject: [PATCH 0334/1239] Update generator_pythia8_LF.C --- MC/config/PWGLF/pythia8/generator_pythia8_LF.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/PWGLF/pythia8/generator_pythia8_LF.C b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C index b5550fb5e..c35c539ad 100644 --- a/MC/config/PWGLF/pythia8/generator_pythia8_LF.C +++ b/MC/config/PWGLF/pythia8/generator_pythia8_LF.C @@ -165,7 +165,7 @@ class GeneratorPythia8LF : public o2::eventgen::GeneratorPythia8 if (mGapBetweenInjection > 0) { if (mGapBetweenInjection == 1 && mEventCounter % 2 == 0) { doSignal = false; - } else if (mEventCounter % mGapBetweenInjection != 0) { + } else if (mEventCounter % mGapBetweenInjection + 1 != 0) { doSignal = false; } } From 94e512ce08d48b7785219eb80d7db90d3be5ae43 Mon Sep 17 00:00:00 2001 From: shahoian Date: Thu, 1 Feb 2024 12:08:53 +0100 Subject: [PATCH 0335/1239] Enable M-shape correction by defualt if any correction is allowed --- .../configurations/asyncReco/setenv_extra.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index d9bb33d4b..4c95be58a 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -315,6 +315,8 @@ elif [[ $ALIGNLEVEL == 1 ]]; then fi # now we set the options + [[ -n "$ALIEN_JDL_MSHAPE_CORRECTION" && $ALIEN_JDL_MSHAPE_CORRECTION == "0" ]] && ENABLE_MSHAPE=0 || ENABLE_MSHAPE=1 + if [[ $INST_IR_FOR_TPC -gt 0 ]]; then # externally imposed IR for scaling echo "Applying externally provided IR for scaling, $INST_IR_FOR_TPC Hz" export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" @@ -324,6 +326,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then elif [[ $INST_IR_FOR_TPC -lt 0 ]]; then # do not apply any correction echo "Passed valued for scaling is smaller than zero, no scaling will be applied" echo "NOTA BENE: In the future, this value will signal to not apply any correction at all, which is not operational yet (but please check, as it depends on O2)" + ENABLE_MSHAPE=0 export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" elif [[ $INST_IR_FOR_TPC == "CTPCCDB" ]]; then # using what we have in the CCDB CTP counters, extracted at the beginning of the script echo "Using CTP CCDB which gave the mean IR of the run at the beginning of the script ($RUN_IR Hz)" @@ -346,6 +349,10 @@ elif [[ $ALIGNLEVEL == 1 ]]; then return 1 fi + if [[ $ENABLE_MSHAPE == "1" ]]; then + export TPC_CORR_SCALING+=" --enable-M-shape-correction " + fi + if [[ -n $ALIEN_JDL_MEANIRFORTPC && $ALIEN_JDL_MEANIRFORTPC > 0 ]]; then # externally imposed TPC map mean IR for scaling export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$ALIEN_JDL_MEANIRFORTPC" fi @@ -363,6 +370,9 @@ elif [[ $ALIGNLEVEL == 1 ]]; then return 1 fi fi + if [[ $ENABLE_MSHAPE == "1" ]]; then + export TPC_CORR_SCALING+=" --enable-M-shape-correction " + fi fi echo "Final setting for TPC scaling is:" From 50ec7dc3ae2ae95327f4787fc7ed74cfaaf4f86d Mon Sep 17 00:00:00 2001 From: swenzel Date: Thu, 1 Feb 2024 13:35:05 +0100 Subject: [PATCH 0336/1239] platform independence of containerized execution --- GRID/utils/runGRIDContainerized.sh | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/GRID/utils/runGRIDContainerized.sh b/GRID/utils/runGRIDContainerized.sh index ca98f616c..1752f692d 100755 --- a/GRID/utils/runGRIDContainerized.sh +++ b/GRID/utils/runGRIDContainerized.sh @@ -7,13 +7,31 @@ SCRIPT=$1 [ $SCRIPT == "" ] && echo "Please provide a script to run" && exit 1 echo "Trying to run script ${SCRIPT} in a container environment" +# detect architecture (ARM or X86) +ARCH=$(uname -i) +if [ "$ARCH" == "aarch64" ] || [ "$arch" == "x86" ]; then + echo "Detected hardware architecture : $ARCH" +else + echo "Invalid architecture ${ARCH} detected. Exiting" + exit 1 +fi +if [ "$ARCH" == "aarch64" ]; then + ISAARCH64="1" +fi # we just use the default singularity container -APPTAINER_CONTAINER=/cvmfs/alice.cern.ch/containers/fs/singularity/default +APPTAINER_CONTAINER=/cvmfs/alice.cern.ch/containers/fs/singularity/default${ISAARCH64+"-aarch64"} -# create workdir -WORK_DIR=$(mktemp -d /tmp/alien-job-XXXXXX) +# create workdir if not specified externally +if [ ! "${WORK_DIR}" ]; then + WORK_DIR=$(mktemp -d /tmp/alien-job-XXXXXX) +fi echo "This job will be run in $WORK_DIR" +if [ ! -d "${WORK_DIR}" ]; then + echo "working directory ${WORK_DIR} does not exist; Please create before running" + exit 1 +fi + # copy script to WORK_DIR cp ${SCRIPT} ${WORK_DIR}/job.sh @@ -31,5 +49,5 @@ echo "JALIEN_TOKEN_CERT=/workdir/usercert.pem" > ${WORK_DIR}/envfile echo "JALIEN_TOKEN_KEY=/workdir/userkey.pem" >> ${WORK_DIR}/envfile # launch job = script inside the container in the workdir -/cvmfs/alice.cern.ch/containers/bin/apptainer/current/bin/apptainer exec -C -B /cvmfs:/cvmfs,${WORK_DIR}:/workdir \ +/cvmfs/alice.cern.ch/containers/bin/apptainer/current${ISAARCH64+"-aarch64"}/bin/apptainer exec -C -B /cvmfs:/cvmfs,${WORK_DIR}:/workdir \ --pwd /workdir --env-file ${WORK_DIR}/envfile ${APPTAINER_CONTAINER} /workdir/job.sh From 6dcf1448255b35273c1c6355d6a2713f5d9029a9 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Fri, 2 Feb 2024 14:08:00 +0100 Subject: [PATCH 0337/1239] TPC QC allow proper init of propagator to avoid error message in FST (#1436) --- DATA/production/qc-sync/tpc.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DATA/production/qc-sync/tpc.json b/DATA/production/qc-sync/tpc.json index 5d0b22997..daabaa239 100644 --- a/DATA/production/qc-sync/tpc.json +++ b/DATA/production/qc-sync/tpc.json @@ -110,6 +110,16 @@ "cutMinNCluster": "60", "cutMindEdxTot": "20." }, + "grpGeomRequest" : { + "geomRequest": "Aligned", + "askGRPECS": "false", + "askGRPLHCIF": "false", + "askGRPMagField": "true", + "askMatLUT": "true", + "askTime": "false", + "askOnceAllButField": "true", + "needPropagatorD": "false" + }, "location": "local", "localMachines": [ "localhost", From 319e7f079bdc3caa38069b4c84c4f11742e4827d Mon Sep 17 00:00:00 2001 From: Daniel Samitz <69901155+DanielSamitz@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:39:36 +0100 Subject: [PATCH 0338/1239] load libpythia6 (#1435) --- MC/config/PWGEM/external/generator/GeneratorEMCocktailV2.C | 1 + 1 file changed, 1 insertion(+) diff --git a/MC/config/PWGEM/external/generator/GeneratorEMCocktailV2.C b/MC/config/PWGEM/external/generator/GeneratorEMCocktailV2.C index 9e2deac45..13770644d 100644 --- a/MC/config/PWGEM/external/generator/GeneratorEMCocktailV2.C +++ b/MC/config/PWGEM/external/generator/GeneratorEMCocktailV2.C @@ -1,4 +1,5 @@ R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGDQ/external/generator) +R__LOAD_LIBRARY(libpythia6) #include "GeneratorCocktail.C" namespace o2 { From 98b4ad04c01a98386578f3b1efbe05f2baf25c93 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 5 Feb 2024 09:23:19 +0100 Subject: [PATCH 0339/1239] Documentation and fixes (#1405) * move documentation to https://aliceo2group.github.io/simulation/docs/relval/ * Adding more comments to sources * re-arrange some code blocks with respect to each other * some bug fixes Co-authored-by: Benedikt Volkel --- RelVal/README.md | 173 +--------- RelVal/o2dpg_release_validation.py | 245 +++++++------- RelVal/utils/o2dpg_release_validation_plot.py | 68 +++- .../utils/o2dpg_release_validation_utils.py | 307 ++++++++++++------ 4 files changed, 394 insertions(+), 399 deletions(-) diff --git a/RelVal/README.md b/RelVal/README.md index b2d1c7d2a..680f9a4db 100644 --- a/RelVal/README.md +++ b/RelVal/README.md @@ -1,174 +1,3 @@ # O2DPG ReleaseValidation (RelVal) -The RelVal is specifically designed to compare 2 sets of QC objects. However, it is also possible to compare ROOT files that contain other objects such as histograms (`TH1`) or also `TTree`s: -* ROOT histograms (deriving from `TH1`) -* ROOT `TProfile` -* ROOT `TEfficiency` -* O2 `o2::quality_control::core::MonitorObjectCollection` -* O2 `o2::quality_control::core::MonitorObject` -* ROOT `TTree` (Here the algorithm does its best to extract as many TLeafs as possible which works when they can be drawn with `TTree::Draw`.) - -Objects from compared files are extracted recursively and so all objects in sub-directories are compared. - -The convention is, that only those objects that have the exact same path are compared to one another so the 2 ROOT files must have the same structure. Note though, that all possible pairs are compared. If there are singular objects in one or the other file, they will be safely ignored. - -At the end of this README are some examples for QC RelVal. - -## Quick start - -To jump right in, please check out [this](#run-for-qc) - -## Definitions - -### Metric -A metric is a way to compare 2 corresponding objects and assign a number to that comparison. There are currently 3 definitions: -1. `chi2`: Chi2 test of compared histograms (see also the [ROOT documentation](https://root.cern.ch/doc/master/classTH1.html#ab7d63c7c177ccbf879b5dc31f2311b27)), -1. `kolmogorov`: shape comparison using Kolmogorov test (see also the [ROOT documentation](https://root.cern.ch/doc/master/classTH1.html#aeadcf087afe6ba203bcde124cfabbee4)), -1. `num_entries`: relative difference in the number of entries. -So for each pair of histograms there can be multiple metrics. - -### Test -A test is the comparison of a computed metric to certain limits (upper,lower). How these limits came about is the property of such a test. For instance, a simple **threshold** test, where lower is better, would mean to have limits of `(, -infty)`. -There can hence be multiple tests for one metric. - -### Interpretation -A test can be assigned an interpretation. There are -1. `GOOD` if a metric passes a test, -1. `WARNING`: if a **non-critical** metric fails a test, -1. `NONCRIT_NC` if the objects could not be compared e.g. due to different binning or axis ranges **and** if the metric is considered **non-critical**, -1. `CRIT_NC` if the histograms could not be compared e.g. due to different binning or axis ranges **and** if the metric is considered **critical**, -1. `BAD` if a test of a amtric fails that is considered **critical** -1. `UNKNOWN` used for instance when a test might have been defined but no metric was passed to be tested. - -## Usage - -The [Python script](o2dpg_release_validation.py) is the entrypoint of the RelVal and it has multiple sub-commands. - -The full help message of this script can be seen by typing -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py [] --help -``` -The wrapper includes 3 different sub-commands for now -1. `rel-val` to steer the RelVal, -1. `inspect` to print histograms of specified severity (if any), -1. `compare` to compare the results of 2 RelVal runs, -1. `print` simply print object names, metric names or test names line-by-line to the command line; convenient to further digest the output, -1. `influx` to convert the summary into a format that can be understood by and sent to an InfluxDB instance. - -Each sub-command can be run with `--help` to see all options/flags. - -### `rel-val` - -If you would like to compare 2 files (or sets of files), simply run -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i -j \ - [--include-dirs ] -``` -It will run the full release validation, dumps plots and further artifacts in the directory `rel_val` and prints a result summary in the terminal. -Via the optional `--include-patterns` a list of patterns can be passed so that only those ROOT sub-directories are taken into consideration which contain at least on of those patters, **Note** though, that regular expressions cannot (yet) be used. - -For the comparison of 2 sets of files this is always the first necessary step and of the most important outputs produced is `rel_val/Summary.json` which contains all the test results. It can be used for further and also more in-depth studies as mentioned in the following. - -There are also various plots created during the RelVal run. For each compared file there are -* overlay plots, 1D and 2D (to be found in the sub directory `overlayPlots`), -* 2D plots summarising the interpretations in a grid (called `SummaryTest.png`), -* pie charts showing the fraction of interpretations per metric (and potentially per test, if there are multiple), -* 1D plots showing the computed value and test means per metric (and potentially per test, if there are multiple). - - -### `inspect` -This command requires that a `rel-val` was run previously which produced a `/Summary.json`. - -Imagine you would like to change or experiment with some settings, e.g. you would like to only take objects with certain names into account or only enable certain metrics etc. These things you like to see reflected in the summary as well as in the produced plots. -This is possible with -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path \ - [--include-patterns ] [--exclude-patterns ] \ - [--enable-metric ] [--disable-metric ] \ - [--interpretations ] \ - [--critical ] \ - [--output|-o ] -``` -All of those options, except for `--include-patterns` and `--exclude-patterns` also work with the `rel-val` command. -The output will by default be written to `rel_val_inspect`. All plots which are produced by the `rel-val` command are produced again for a potential given sub-set depending on the given options. Only the overlay plots are not produced again. - -**NOTE** that with `inspect` the original overlay plots satisfying your selection criteria (e.g. `--include-patters` or `--interpretations`) are also copied over to the target directory. - -**Other additional optional arguments** -* `--use-values-as-thresholds []`: By passing a set of summaries that where produced from `rel-val`, the computed metric values can be used as **new** thresholds. To decide how to combine the values for multiple metrics referring to the same object, the option `--combine-thresholds mean|extreme` can be used. Also, an additional relative margin can be added for each metric with `--margin-threshold `; this argument must be repeated for if it should be used for multiple metrics. -* `--regions []`: This computes means and standard deviations for each metric from previously computed values. The corresponding test is passed, if the value lies around the mean within the standard deviations. The deviation from the mean is also given as number-of-sigmas in the summary grid. -* `rel-val -i -j --no-extract` runs RelVal on **flat** ROOT files that have only histogram objects in them. - -### `print` -This command has the same optional arguments as the `inspect` command. But the only thing it does is writing some information line-by-line. For instance, to get the object names that were flagged `BAD` by the `chi2` metric, do -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py print --path --enable-metric chi2 --interpretations BAD -``` -If no RelVal was run but one would like to know the available metrics, one can check with -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py print --metric-names -``` - -### `influx` - -To convert the final output to something that can be digested by InfluxDB, use -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py influx --dir [--tags k1=v1 k2=v2 ...] [--table-name ] -``` -When the `--tags` argument is specified, these are injected as TAGS for InfluxDB in addition. The table name can also be specified explicitly; if not given, it defaults to `O2DPG_MC_ReleaseValidation`. - -## RelVal for QC (examples) - -### Comparing data with MC - -There is an ongoing effort to unify the names of QC objects inside MC and data QC files. Some are already unified and the following command would run comparison of those. However, others are not yet unified and will not be considered in the comparison. - -MC QC objects are usually distributed over multiple files while those from data are all contained in one single file. It is possible to directly compare them with -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i ${MC_PRODUCTION}/QC/*.root -j ${DATA_PRODUCTION}/QC.root [--include-dirs ] -``` - -## Run for QC -This is a simple guide to run RelVal for QC. - -Here is also a [working example](run/run_data_rel_val.sh), run it with -```bash -${O2DPG_ROOT}/RelVal/run/run_data_rel_val.sh [--qc QC1.root QC2.root ] [--aod AOD1.root AOD2.root] [ --labels LABEL1 LABEL2] -``` - -### If you are interested in all QC plots -To have everything and to use this as a starting point for deeper inspections, first run -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i QC_file_1.root -j QC_file_2.root -o rel_val_all [--labels meaningfulLabel1 meaningfulLabel2] -``` -Now, there is of course a lot but from now on you are fully flexible. - -In order to get some insight into a specific detector, say ITS, run -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py inspect --path rel_val_all --include-patterns "^ITS_" -o rel_val_ITS -``` -This will only print pie charts and summaries for ITS and also copies all overlay plots related to ITS to your target directory `rel_val_ITS`. - -The `inspect` command is much faster now since no new plots are generated and metrics do not have to be recomputed. It simply filters the results according to your criteria. However, what can be re-evaluated are the computed values against new thresholds. - -### If you are only interested in some ROOT sub-directories to begin with -If you only want to study for instance the ITS and CPV and there is no interest at this point to study any other detector, run -```bash -${O2DPG_ROOT}/RelVal/o2dpg_release_validation.py rel-val -i QC_file_1.root -j QC_file_2.root -o rel_val_all --include-dirs ITS CPV [--labels meaningfulLabel1 meaningfulLabel2] -``` -From here on, you can use the `inspect` command as usual. But there will never be detectors other than ITS and CPV. - -### Troubleshooting - -If there are unexpected segmentation faults or similar, most likely the `QualityControl` software is not properly linked against `O2`. Most likely, the reason is that `QC` was not rebuild against the loaded `O2` version. -The easiest solution would be to load either `QualityControl` or meta packages such as `O2sim`. -Loading like `O2/latest,QualityControl/latest` can cause problems depending on how the single packages were build. - -## Expert section - -### Adding a new metric -A new metric can be added in [ReleaseValidationMetrics.C](ReleaseValidationMetrics.C) by extending the function `void initialiseMetrics(MetricRunner& metricRunner)`. - -## Future plans - -* Store a JSON/JSONs on CCDB for central derivation of more refined thresholds or regions. +Please find the detailed documentation at [https://aliceo2group.github.io/simulation/docs/relval/](https://aliceo2group.github.io/simulation/docs/relval/). diff --git a/RelVal/o2dpg_release_validation.py b/RelVal/o2dpg_release_validation.py index c776bd624..5904a020a 100755 --- a/RelVal/o2dpg_release_validation.py +++ b/RelVal/o2dpg_release_validation.py @@ -11,11 +11,12 @@ import sys import argparse import importlib.util -from os import environ, makedirs, remove, rename +from os import environ, makedirs, remove from os.path import join, abspath, exists, dirname, basename, isfile -from shutil import copy, rmtree import json +import numpy as np + # make sure O2DPG + O2 is loaded O2DPG_ROOT=environ.get('O2DPG_ROOT') @@ -35,13 +36,13 @@ o2dpg_release_validation_utils = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_utils) sys.modules["o2dpg_release_validation_utils"] = o2dpg_release_validation_utils -from o2dpg_release_validation_utils import * +import o2dpg_release_validation_utils as utils spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_plot.py')) o2dpg_release_validation_plot = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation_plot) sys.modules["o2dpg_release_validation_plot"] = o2dpg_release_validation_plot -from o2dpg_release_validation_plot import plot_pie_charts, plot_summary_grid, plot_compare_summaries, plot_overlays +from o2dpg_release_validation_plot import plot_pie_charts, plot_summary_grid, plot_compare_summaries, plot_overlays, plot_value_histograms ROOT_MACRO_EXTRACT=join(O2DPG_ROOT, "RelVal", "utils", "ExtractAndFlatten.C") @@ -52,45 +53,10 @@ gROOT.SetBatch() -def copy_overlays(rel_val, input_dir, output_dir): - """ - copy overlay plots in this summary from the input directory to the output directory - """ - input_dir = abspath(input_dir) - output_dir = abspath(output_dir) - - if not exists(input_dir): - print(f"ERROR: Input directory {input_dir} does not exist") - return 1 - - inOutSame = input_dir == output_dir - - input_dir_new = input_dir + "_tmp" - if inOutSame: - # move input directory - rename(input_dir, input_dir_new) - input_dir = input_dir_new - - if not exists(output_dir): - makedirs(output_dir) - - object_names, _ = rel_val.get_result_per_metric_and_test() - object_names = list(set(object_names)) - - ret = 0 - for object_name in object_names: - filename=join(input_dir, f"{object_name}.png") - if exists(filename): - copy(filename, output_dir) - else: - print(f"File {filename} not found.") - ret = 1 - - if inOutSame: - rmtree(input_dir) - - return ret +############################################# +# Helper functions only used in this script # +############################################# def metrics_from_root(): """ @@ -100,7 +66,7 @@ def metrics_from_root(): if exists(log_file_name): remove(log_file_name) cmd = f"root -l -b -q {ROOT_MACRO_METRICS}" - ret = run_macro(cmd, log_file_name) + ret = utils.run_macro(cmd, log_file_name) if ret > 0: return ret @@ -116,13 +82,30 @@ def metrics_from_root(): return 0 -def extract(input_filenames, target_filename, include_file_directories=None, add_if_exists=False, reference_extracted=None, json_extracted=None): +def load_from_meta_json(json_path): + """ + Load a meta JSON file and return dictionary + """ + if not exists(json_path): + return None + + with open(json_path, "r") as f: + try: + return json.load(f) + except (json.decoder.JSONDecodeError, UnicodeDecodeError): + pass + return None + + +def extract_and_flatten_impl(input_filenames, target_filename, include_file_directories=None, add_if_exists=False, reference_extracted="", json_extracted=""): """ Wrap the extraction of objects to be compared Will be extracted (from TH1, QC objects, TTree etc.), converted to TH1 and put into a flat ROOT file structure. Args: + input_filenames: list + list of input filenames to extract objects from target_filename: str path to file where extracted objects should be saved include_file_directories: iterable or "" (default: "") @@ -132,15 +115,15 @@ def extract(input_filenames, target_filename, include_file_directories=None, add reference_extracted: str is used in case of the extraction of TTrees in which case the x-axis binning will be set according to that reference to make objects comparable. + json_extracted: str + the path to where the JSON file with the info of "what has been extracted where" will be saved + Returns: - bool - True in case of success, False otherwise + bool: True in case of success, False otherwise """ def get_files_from_list(list_filename): """ - Quick helper - - Extract filenames from what is listed in a given file + Quick helper to extract filenames from what is listed in a given file """ collect_files = [] with open(list_filename, "r") as f: @@ -153,13 +136,13 @@ def get_files_from_list(list_filename): include_file_directories = ",".join(include_file_directories) if include_file_directories else "" - # flat ROOT files to extract to and read from during RelVal; make absolute paths so we don't confuse ourselves when running e.g. ROOT macros in different directories - if len(input_filenames) == 1 and input_filenames[0][0] == "@": - input_filenames = get_files_from_list(input_filenames[0][1:]) - if not files1: - print(f"ERROR: Apparently {input_filenames[0][1:]} contains no files to be extracted.") - return None + # if there is only one filename and it starts with "@", assume that it contains the paths of the actual files that should be extracted + read_files_from = input_filenames[0][1:] + input_filenames = get_files_from_list(read_files_from) + if not input_filenames: + print(f"ERROR: Apparently {read_files_from} contains no files to be extracted.") + return False if exists(target_filename) and not add_if_exists: # this file will otherwise be updated if it exists @@ -169,10 +152,6 @@ def get_files_from_list(list_filename): cwd = dirname(target_filename) target_filename = basename(target_filename) log_file_name = join(cwd, f"{target_filename}_extract_and_flatten.log") - if not reference_extracted: - reference_extracted = "" - if not json_extracted: - json_extracted = "" print("Extraction of files") @@ -181,30 +160,23 @@ def get_files_from_list(list_filename): print(f" {f}") cmd = f"\\(\\\"{f}\\\",\\\"{target_filename}\\\",\\\"{reference_extracted}\\\",\\\"{include_file_directories}\\\",\\\"{json_extracted}\\\"\\)" cmd = f"root -l -b -q {ROOT_MACRO_EXTRACT}{cmd}" - ret = run_macro(cmd, log_file_name, cwd) + ret = utils.run_macro(cmd, log_file_name, cwd) if ret != 0: print(f"ERROR: Extracting from file {f} failed. Please check logfile {abspath(join(cwd, log_file_name))}") return False return True -def get_extract_json_info(json_path): - - if not exists(json_path): - return None - - with open(json_path, "r") as f: - try: - return json.load(f) - except (json.decoder.JSONDecodeError, UnicodeDecodeError): - pass - return None +def extract_and_flatten(files, output, label, include_directories=None, add_if_exists=False, prefix=None, reference_extracted=""): + """ + Extract from input files to a flat ROOT file -def only_extract_impl(files, output, label, include_directories=None, add_if_exists=False, prefix=None, reference_extracted=None): + Returns the path to a meta JSON and that JSON file loaded as dictionary + """ if len(files) == 1: - d = get_extract_json_info(files[0]) + d = load_from_meta_json(files[0]) if d is not None: return files[0], d @@ -216,7 +188,7 @@ def only_extract_impl(files, output, label, include_directories=None, add_if_exi json_out = abspath(join(output, json_out)) root_out = abspath(join(output, root_out)) - if not extract(files, root_out, include_file_directories=include_directories, add_if_exists=add_if_exists, reference_extracted=reference_extracted, json_extracted=json_out): + if not extract_and_flatten_impl(files, root_out, include_file_directories=include_directories, add_if_exists=add_if_exists, reference_extracted=reference_extracted, json_extracted=json_out): return None, None d = None @@ -231,12 +203,6 @@ def only_extract_impl(files, output, label, include_directories=None, add_if_exi return json_out, d -def only_extract(args): - if not only_extract_impl(args.input, args.output, None, args.label, args.reference): - return 1 - return 0 - - def rel_val_root(d1, d2, metrics_enabled, metrics_disabled, output_dir): """ RelVal for 2 ROOT files, simply a wrapper around ReleaseValidation.C macro @@ -280,7 +246,7 @@ def rel_val_root(d1, d2, metrics_enabled, metrics_disabled, output_dir): output_dir = abspath(output_dir) log_file_rel_val = join(output_dir, "rel_val.log") print("Running RelVal on extracted objects") - ret = run_macro(cmd, log_file_rel_val, cwd=output_dir) + ret = utils.run_macro(cmd, log_file_rel_val, cwd=output_dir) # This comes from the ROOT macro json_path = join(output_dir, "RelVal.json") @@ -309,7 +275,7 @@ def load_rel_val(json_path, include_patterns=None, exclude_patterns=None, enable Returns RelVal """ - rel_val = RelVal() + rel_val = utils.RelVal() rel_val.set_object_name_patterns(include_patterns, exclude_patterns) rel_val.enable_metrics(enable_metrics) rel_val.disable_metrics(disable_metrics) @@ -317,14 +283,14 @@ def load_rel_val(json_path, include_patterns=None, exclude_patterns=None, enable return rel_val -def initialise_evaluator(rel_val, thresholds, thresholds_default, thresholds_margins, thresholds_combine, regions): +def initialise_evaluator(rel_val, thresholds_paths, thresholds_default, thresholds_margins, thresholds_combine, regions_paths): """ Wrapper to create an evaluator Args: rel_val: RelVal the RelVal object that should potentially be tested and is used to derive default threshold - thresholds: iterable or None + thresholds_paths: iterable or None if not None, iterable of string as the paths to RelVal JSONs thresholds_defaults: iterable of 2-tuples or None assign a default threshold value (tuple[1]) to a metric name (tuple[0]) @@ -332,29 +298,44 @@ def initialise_evaluator(rel_val, thresholds, thresholds_default, thresholds_mar add a margin given in percent (tuple[1]) to a threshold value of a metric name (tuple[0]) thresholds_combine: str either "mean" or "extreme", how threshold values extracted from argument thresholds should be combined - regions: iterable or None + regions_paths: iterable or None if not None, iterable of string as the paths to RelVal JSONs Returns: Evaluator """ - evaluator = Evaluator() + evaluator = utils.Evaluator() # initialise to run tests on proper mean +- std - if regions: - rel_val_regions = get_summaries_or_from_file(regions) - initialise_regions(evaluator, rel_val_regions) + if regions_paths: + regions = utils.get_paths_or_from_file(regions_paths) + rel_val_regions = utils.RelVal() + rel_val_regions.load(regions) + utils.initialise_regions(evaluator, rel_val_regions) # initialise to run tests on thresholds thresholds_default = {metric_name: float(value) for metric_name, value in thresholds_default} if thresholds_default else None rel_val_thresholds = None - if thresholds: + if thresholds_paths: thresholds_margins = {metric_name: float(value) for metric_name, value in thresholds_margins} if thresholds_margins else None - rel_val_thresholds = get_summaries_or_from_file(thresholds) - initialise_thresholds(evaluator, rel_val, rel_val_thresholds, thresholds_default, thresholds_margins, thresholds_combine) + thresholds_paths = utils.get_paths_or_from_file(thresholds_paths) + rel_val_thresholds = utils.RelVal() + rel_val_thresholds.load(thresholds_paths) + utils.initialise_thresholds(evaluator, rel_val, rel_val_thresholds, thresholds_default, thresholds_margins, thresholds_combine) evaluator.initialise() return evaluator +################################################################### +# Functions that are called after command line has been processed # +################################################################### + + +def only_extract(args): + if not extract_and_flatten(args.input, args.output, None, args.label, args.reference)[0]: + # checking one of the return values for None + return 1 + return 0 + def rel_val(args): """ @@ -364,7 +345,7 @@ def rel_val(args): """ def interpret_results(result, metric): """ - Taking in a result and the metric it was derived from, assign an interpretation + Taking in a result and the corresponding metric it was derived from and assign an interpretation """ is_critical = args.is_critical is None or metric.name in args.is_critical if not metric.comparable and is_critical: @@ -373,13 +354,13 @@ def interpret_results(result, metric): if not metric.comparable: result.interpretation = variables.REL_VAL_INTERPRETATION_NONCRIT_NC return - if result.result_flag == Result.FLAG_UNKNOWN: + if result.result_flag == utils.Result.FLAG_UNKNOWN: result.interpretation = variables.REL_VAL_INTERPRETATION_UNKNOWN return - if result.result_flag == Result.FLAG_PASSED: + if result.result_flag == utils.Result.FLAG_PASSED: result.interpretation = variables.REL_VAL_INTERPRETATION_GOOD return - if result.result_flag == Result.FLAG_FAILED and is_critical: + if result.result_flag == utils.Result.FLAG_FAILED and is_critical: result.interpretation = variables.REL_VAL_INTERPRETATION_BAD return result.interpretation = variables.REL_VAL_INTERPRETATION_WARNING @@ -389,38 +370,42 @@ def interpret_results(result, metric): need_apply = False is_inspect = False - json1 = None - json2 = None + dict_1 = None + dict_2 = None if hasattr(args, "json_path"): # this comes from the inspect command is_inspect = True - json_path = get_summary_path(args.json_path) + json_path = utils.get_summary_path(args.json_path) annotations = None include_patterns, exclude_patterns = (args.include_patterns, args.exclude_patterns) else: # in this case, new input ROOT files were provided and we need to apply all our tests need_apply = True + # always take everything include_patterns, exclude_patterns = (None, None) if args.add: print(f"NOTE: Extracted objects will be added to existing ones in case there was already a RelVal at {args.output}.\n") - json1 = only_extract_impl(args.input1, args.output, args.labels[0], args.include_dirs, args.add, prefix="1", reference_extracted=None) - json2 = only_extract_impl(args.input2, args.output, args.labels[1], args.include_dirs, args.add, prefix="2", reference_extracted=json1[1]["path"]) - if None in json1 or None in json2: - print("ERROR: Something went wrong during the extraction") + # each extraction will leave us with a JSON + json_path_1, dict_1 = extract_and_flatten(args.input1, args.output, args.labels[0], args.include_dirs, args.add, prefix="1", reference_extracted=None) + if not json_path_1: + return 1 + json_path_2, dict_2 = extract_and_flatten(args.input2, args.output, args.labels[1], args.include_dirs, args.add, prefix="2", reference_extracted=dict_1["path"]) + if not json_path_2: return 1 - json_path = rel_val_root(json1[1], json2[1], args.enable_metric, args.disable_metric, args.output) + json_path = rel_val_root(dict_1, dict_2, args.enable_metric, args.disable_metric, args.output) if json_path is None: print("ERROR: Problem during RelVal") return 1 - annotations = {"json_path_1": json1[0], - "json_path_2": json2[0]} + annotations = {"json_path_1": json_path_1, + "json_path_2": json_path_2} + # now loading and constructing a RelVal object rel_val = load_rel_val(json_path, include_patterns, exclude_patterns, args.enable_metric, args.disable_metric) if need_apply or args.use_values_as_thresholds or args.default_threshold or args.regions: evaluator = initialise_evaluator(rel_val, args.use_values_as_thresholds, args.default_threshold, args.margin_threshold, args.combine_thresholds, args.regions) - rel_val.apply(evaluator) + rel_val.apply_evaluator(evaluator) # assign interpretations to the results we got rel_val.interpret(interpret_results) @@ -435,7 +420,7 @@ def filter_on_interpretations(result): # if this comes from inspecting, there will be the annotations from the rel-val before that ==> re-write it rel_val.write(join(args.output, "Summary.json"), annotations=annotations or rel_val.annotations[0]) - print_summary(rel_val, variables.REL_VAL_SEVERITIES, long=args.print_long) + utils.print_summary(rel_val, variables.REL_VAL_SEVERITIES, long=args.print_long) if not args.no_plot: print("Now plotting...") @@ -443,21 +428,19 @@ def filter_on_interpretations(result): plot_pie_charts(rel_val, variables.REL_VAL_SEVERITIES, variables.REL_VAL_SEVERITY_COLOR_MAP, args.output) plot_compare_summaries((rel_val,), args.output) plot_summary_grid(rel_val, variables.REL_VAL_SEVERITIES, variables.REL_VAL_SEVERITY_COLOR_MAP, args.output) + plot_value_histograms(rel_val, args.output) if is_inspect: if annotations_inspect := rel_val.annotations: annotations_inspect = annotations_inspect[0] - d1 = get_extract_json_info(annotations_inspect["json_path_1"]) - d2 = get_extract_json_info(annotations_inspect["json_path_2"]) - else: - d1 = json1[1] - d2 = json2[1] + dict_1 = load_from_meta_json(annotations_inspect["json_path_1"]) + dict_2 = load_from_meta_json(annotations_inspect["json_path_2"]) - if d1 and d2: + if dict_1 and dict_2: overlay_plots_out = join(args.output, "overlayPlots") if not exists(overlay_plots_out): makedirs(overlay_plots_out) - plot_overlays(rel_val, d1, d2, overlay_plots_out) + plot_overlays(rel_val, dict_1, dict_2, overlay_plots_out) return 0 @@ -473,8 +456,8 @@ def compare(args): output_dir = args.output # load - rel_val1 = load_rel_val(get_summary_path(args.input1[0]), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) - rel_val2 = load_rel_val(get_summary_path(args.input2[0]), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) + rel_val1 = load_rel_val(utils.get_summary_path(args.input1[0]), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) + rel_val2 = load_rel_val(utils.get_summary_path(args.input2[0]), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) # get the test and metric names they have in common test_names = np.intersect1d(rel_val1.known_test_names, rel_val2.known_test_names) @@ -490,8 +473,8 @@ def compare(args): if args.interpretations and interpretation not in args.interpretations: continue # object names of Results matching an interpretation - object_names_interpretation1 = object_names1[count_interpretations(results1, interpretation)] - object_names_interpretation2 = object_names2[count_interpretations(results2, interpretation)] + object_names_interpretation1 = object_names1[utils.count_interpretations(results1, interpretation)] + object_names_interpretation2 = object_names2[utils.count_interpretations(results2, interpretation)] # elements in 1 that are not in 2... only_in1 = np.setdiff1d(object_names_interpretation1, object_names_interpretation2) # ...and the other way round @@ -519,7 +502,7 @@ def influx(args): """ Create an influxDB metrics file """ - rel_val = load_rel_val(get_summary_path(args.path)) + rel_val = load_rel_val(utils.get_summary_path(args.path)) output_path = args.path if isfile(args.path) else join(args.path, "influxDB.dat") table_name = "O2DPG_MC_ReleaseValidation" @@ -566,7 +549,7 @@ def print_simple(args): return 0 return metrics_from_root() - rel_val = load_rel_val(get_summary_path(args.path), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) + rel_val = load_rel_val(utils.get_summary_path(args.path), args.include_patterns, args.exclude_patterns, args.enable_metric, args.disable_metric) def filter_on_interpretations(result): # only consider those results that match a flag requested by the user @@ -594,14 +577,18 @@ def print_header(): print(f"\n{'#' * 25}\n#{' ' * 23}#\n# RUN ReleaseValidation #\n#{' ' * 23}#\n{'#' * 25}\n") -# we define the parser here +################################################################ +# define the parser globally so that it could even be imported # +################################################################ +# common parser for digesting input files COMMON_FILE_PARSER = argparse.ArgumentParser(add_help=False) COMMON_FILE_PARSER.add_argument("-i", "--input1", nargs="*", help="EITHER first set of input files for comparison OR first input directory from simulation for comparison", required=True) COMMON_FILE_PARSER.add_argument("-j", "--input2", nargs="*", help="EITHER second set of input files for comparison OR second input directory from simulation for comparison", required=True) COMMON_FILE_PARSER.add_argument("--labels", nargs=2, help="labels you want to appear in the plot legends in case of overlay plots from batches -i and -j", default=("batch_i", "batch_j")) COMMON_FILE_PARSER.add_argument("--no-extract", dest="no_extract", action="store_true", help="no extraction but immediately expect histograms present for comparison") +# common parser digesting options related to thresholds COMMON_THRESHOLD_PARSER = argparse.ArgumentParser(add_help=False) COMMON_THRESHOLD_PARSER.add_argument("--regions", help="Use calculated regions to test status") COMMON_THRESHOLD_PARSER.add_argument("--default-threshold", dest="default_threshold", action="append", nargs=2) @@ -609,41 +596,53 @@ def print_header(): COMMON_THRESHOLD_PARSER.add_argument("--combine-thresholds", dest="combine_thresholds", choices=["mean", "extreme"], help="Arithmetic mean or extreme value is chosen as threshold", default="mean") COMMON_THRESHOLD_PARSER.add_argument("--margin-threshold", dest="margin_threshold", action="append", nargs=2) +# common parser to digest metric options COMMON_METRIC_PARSER = argparse.ArgumentParser(add_help=False) COMMON_METRIC_PARSER.add_argument("--enable-metric", dest="enable_metric", nargs="*") COMMON_METRIC_PARSER.add_argument("--disable-metric", dest="disable_metric", nargs="*") +# common parser to digest object name patterns COMMON_PATTERN_PARSER = argparse.ArgumentParser(add_help=False) COMMON_PATTERN_PARSER.add_argument("--include-patterns", dest="include_patterns", nargs="*", help="include objects whose name includes at least one of the given patterns (takes precedence)") COMMON_PATTERN_PARSER.add_argument("--exclude-patterns", dest="exclude_patterns", nargs="*", help="exclude objects whose name includes at least one of the given patterns") +# common parser to digest options related to interpretations COMMON_FLAGS_PARSER = argparse.ArgumentParser(add_help=False) COMMON_FLAGS_PARSER.add_argument("--interpretations", nargs="*", help="extract all objects which have at least one test with this severity flag", choices=list(variables.REL_VAL_SEVERITY_MAP.keys())) COMMON_FLAGS_PARSER.add_argument("--is-critical", dest="is_critical", nargs="*", help="set names of metrics that are assumed to be critical") +# common parser to handle verbosity COMMON_VERBOSITY_PARSER = argparse.ArgumentParser(add_help=False) COMMON_VERBOSITY_PARSER.add_argument("--print-long", dest="print_long", action="store_true", help="enhance verbosity") COMMON_VERBOSITY_PARSER.add_argument("--no-plot", dest="no_plot", action="store_true", help="suppress plotting") +# The main parser PARSER = argparse.ArgumentParser(description='Wrapping ReleaseValidation macro') + +# Use various sub-parsers SUB_PARSERS = PARSER.add_subparsers(dest="command") + +# rel-val REL_VAL_PARSER = SUB_PARSERS.add_parser("rel-val", parents=[COMMON_FILE_PARSER, COMMON_METRIC_PARSER, COMMON_THRESHOLD_PARSER, COMMON_FLAGS_PARSER, COMMON_VERBOSITY_PARSER]) REL_VAL_PARSER.add_argument("--include-dirs", dest="include_dirs", nargs="*", help="only include desired directories inside ROOT file; note that each pattern is assumed to start in the top-directory (at the moment no regex or *)") REL_VAL_PARSER.add_argument("--add", action="store_true", help="If given and there is already a RelVal in the output directory, extracted objects will be added to the existing ones") REL_VAL_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val") REL_VAL_PARSER.set_defaults(func=rel_val) +# inspect INSPECT_PARSER = SUB_PARSERS.add_parser("inspect", parents=[COMMON_THRESHOLD_PARSER, COMMON_METRIC_PARSER, COMMON_PATTERN_PARSER, COMMON_FLAGS_PARSER, COMMON_VERBOSITY_PARSER]) INSPECT_PARSER.add_argument("--path", dest="json_path", help="either complete file path to a Summary.json or directory where one of the former is expected to be", required=True) INSPECT_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val_inspect") INSPECT_PARSER.set_defaults(func=rel_val) +# compare COMPARE_PARSER = SUB_PARSERS.add_parser("compare", parents=[COMMON_FILE_PARSER, COMMON_PATTERN_PARSER, COMMON_METRIC_PARSER, COMMON_VERBOSITY_PARSER, COMMON_FLAGS_PARSER]) COMPARE_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val_comparison") COMPARE_PARSER.add_argument("--difference", action="store_true", help="plot histograms with different severity") COMPARE_PARSER.add_argument("--plot", action="store_true", help="plot value and threshold comparisons of RelVals") COMPARE_PARSER.set_defaults(func=compare) +# influx INFLUX_PARSER = SUB_PARSERS.add_parser("influx") INFLUX_PARSER.add_argument("--path", help="directory where ReleaseValidation was run", required=True) INFLUX_PARSER.add_argument("--tags", nargs="*", help="tags to be added for influx, list of key=value") @@ -651,6 +650,7 @@ def print_header(): INFLUX_PARSER.add_argument("--output", "-o", help="output path; if not given, a file influxDB.dat is places inside the RelVal directory") INFLUX_PARSER.set_defaults(func=influx) +# print PRINT_PARSER = SUB_PARSERS.add_parser("print", parents=[COMMON_METRIC_PARSER, COMMON_PATTERN_PARSER, COMMON_FLAGS_PARSER]) PRINT_PARSER.add_argument("--path", help="either complete file path to a Summary.json or directory where one of the former is expected to be") PRINT_PARSER.add_argument("--metric-names", dest="metric_names", action="store_true") @@ -658,6 +658,7 @@ def print_header(): PRINT_PARSER.add_argument("--object-names", dest="object_names", action="store_true") PRINT_PARSER.set_defaults(func=print_simple) +# extract EXTRACT_PARSER = SUB_PARSERS.add_parser("extract", parents=[COMMON_VERBOSITY_PARSER]) EXTRACT_PARSER.add_argument("--input", nargs="*", help="Set of input files to be extracted", required=True) EXTRACT_PARSER.add_argument("--output", "-o", help="output directory", default="rel_val_extracted") diff --git a/RelVal/utils/o2dpg_release_validation_plot.py b/RelVal/utils/o2dpg_release_validation_plot.py index 17b774dab..562f5446d 100755 --- a/RelVal/utils/o2dpg_release_validation_plot.py +++ b/RelVal/utils/o2dpg_release_validation_plot.py @@ -7,11 +7,11 @@ from os import environ import importlib.util from itertools import product -import re import numpy as np import matplotlib.pyplot as plt from matplotlib.colors import LinearSegmentedColormap import seaborn +from scipy.stats import iqr O2DPG_ROOT = environ.get("O2DPG_ROOT") @@ -27,13 +27,19 @@ sys.modules["o2dpg_release_validation_plot_root"] = o2dpg_release_validation_plot_root from o2dpg_release_validation_plot_root import plot_overlays_root, plot_overlays_root_no_rel_val + def plot_pie_charts(rel_val, interpretations, interpretation_colors, out_dir, title="", get_figure=False): + """ + Plot pie charts per metric and test + Each pie chart shows the ratio of given interpretations + """ print("==> Plot pie charts <==") for metric_name, test_name in product(rel_val.known_metrics, rel_val.known_test_names): figure, ax = plt.subplots(figsize=(20, 20)) - colors = [] + # collect counts of interpretations, their colours and labels counts = [] + colors = [] labels = [] object_names, results = rel_val.get_result_per_metric_and_test(metric_name, test_name) @@ -60,7 +66,47 @@ def plot_pie_charts(rel_val, interpretations, interpretation_colors, out_dir, ti plt.close(figure) +def plot_value_histograms(rel_val, out_dir, title="values histogram", get_figure=False): + """ + Plot a histogram of metric values + """ + + print("==> Plot value histograms <==") + for metric_name in rel_val.known_metrics: + figure, ax = plt.subplots(figsize=(20, 20)) + values = [] + for _, _, metric in zip(*rel_val.get_metrics(metric_name=metric_name)): + if not metric.comparable: + continue + values.append(metric.value) + + if not values: + continue + + ax.set_xlabel(metric_name, fontsize=20) + ax.set_ylabel("counts", fontsize=20) + ax.hist(values, bins=100) + ax.tick_params("both", labelsize=20) + figure.tight_layout() + + figure.suptitle(f"{title} (metric: {metric_name})", fontsize=40) + save_path = join(out_dir, f"histogram_values_{metric_name}.png") + figure.savefig(save_path) + if get_figure: + return figure + plt.close(figure) + + def plot_summary_grid(rel_val, interpretations, interpretation_colors, output_dir, get_figure=False): + """ + Plot a summary grid per test. + + horizontal axis: metric names + vertical axis: object names + + Each cell is coloured according to an interpretation. + In addition, the cells contain the computed metric values + """ print("==> Plot summary grid <==") @@ -74,12 +120,15 @@ def plot_summary_grid(rel_val, interpretations, interpretation_colors, output_di for nt in range(rel_val.number_of_tests): metric_names, object_names, results_matrix = rel_val.get_result_matrix_objects_metrics(nt) - arr = np.full(results_matrix.shape, 0, dtype=int) + # make an array where each interpretation is mapped to a numerical value + arr_interpretation = np.full(results_matrix.shape, 0, dtype=int) + # collect annotations for each cell arr_annot = np.full(results_matrix.shape, "", dtype=object) + # iterate over the cells and set values and annotations it = np.nditer(results_matrix, flags=['multi_index', "refs_ok"]) for _ in it: result = results_matrix[it.multi_index] - arr[it.multi_index] = interpretation_name_to_number[result.interpretation] + arr_interpretation[it.multi_index] = interpretation_name_to_number[result.interpretation] if result.value is not None: annot = f"{result.value:.3f} (mean: {result.mean:.3f})" if result.n_sigmas is not None: @@ -89,8 +138,9 @@ def plot_summary_grid(rel_val, interpretations, interpretation_colors, output_di arr_annot[it.multi_index] = annot + #now comes the plotting figure, ax = plt.subplots(figsize=(20, 20)) - seaborn.heatmap(arr, ax=ax, cmap=cmap, vmin=-0.5, vmax=len(interpretations) - 0.5, yticklabels=object_names, xticklabels=metric_names, linewidths=0.5, annot=arr_annot, fmt="") + seaborn.heatmap(arr_interpretation, ax=ax, cmap=cmap, vmin=-0.5, vmax=len(interpretations) - 0.5, yticklabels=object_names, xticklabels=metric_names, linewidths=0.5, annot=arr_annot, fmt="") cbar = ax.collections[0].colorbar cbar.set_ticks(range(len(colors))) cbar.set_ticklabels(interpretations) @@ -111,7 +161,7 @@ def plot_summary_grid(rel_val, interpretations, interpretation_colors, output_di def plot_compare_summaries(rel_vals, out_dir, *, labels=None, get_figure=False): """ - if labels is given, it needs to have the same length as summaries + Plot the metric values for each object. """ print("==> Plot metric values <==") @@ -156,10 +206,16 @@ def plot_compare_summaries(rel_vals, out_dir, *, labels=None, get_figure=False): def plot_overlays(rel_val, file_config_map1, file_config_map2, out_dir, plot_regex=None): + """ + Wrapper around ROOT overlay plotting + """ print("==> Plot overlays <==") plot_overlays_root(rel_val, file_config_map1, file_config_map2, out_dir, plot_regex) def plot_overlays_no_rel_val(file_configs, out_dir): + """ + Wrapper around ROOT plotting when no RelVal object is given + """ print("==> Plot overlays <==") plot_overlays_root_no_rel_val(file_configs, out_dir) diff --git a/RelVal/utils/o2dpg_release_validation_utils.py b/RelVal/utils/o2dpg_release_validation_utils.py index 873d4de44..633e424c6 100755 --- a/RelVal/utils/o2dpg_release_validation_utils.py +++ b/RelVal/utils/o2dpg_release_validation_utils.py @@ -3,7 +3,9 @@ # Definition of common functionality import re -from os.path import join, exists, isdir +from os.path import join, exists, isdir, abspath +from os import makedirs, rename +from shutil import rmtree, copy from itertools import product from subprocess import Popen, PIPE, STDOUT from shlex import split @@ -11,20 +13,6 @@ import numpy as np -def remove_outliers(data, m=6.): - """ - Helper to remove outliers from a list of floats - """ - if not data: - return None, None - data = np.array(data) - d = np.abs(data - np.median(data)) - mdev = np.median(d) - s = d / (mdev if mdev else 1.) - print(s) - return data[s < m], data[s >= m] - - def default_evaluation(limits): """ Return a lambda f(value) -> bool @@ -34,10 +22,10 @@ def default_evaluation(limits): if limits[0] is None and limits[1] is None: return lambda x: None if limits[0] is not None and limits[1] is None: - return lambda x: x > limits[0] + return lambda x: x >= limits[0] if limits[0] is None and limits[1] is not None: - return lambda x: x < limits[1] - return lambda x: limits[0] < x < limits[1] + return lambda x: x <= limits[1] + return lambda x: limits[0] <= x <= limits[1] def compute_limits(mean, std): @@ -106,6 +94,9 @@ def __init__(self, object_name=None, name=None, value=None, proposed_threshold=N if in_dict is not None: self.from_dict(in_dict) + def __eq__(self, other): + return self.object_name == other.object_name and self.name == other.name + def as_dict(self): return {"object_name": self.object_name, "metric_name": self.name, @@ -190,36 +181,6 @@ def initialise(self): self.metric_names = np.array(self.metric_names, dtype=str) self.test_names = np.array(self.test_names, dtype=str) self.tests = np.array(self.tests, dtype=TestLimits) - - # fill up tests - # The following guarantees that we have all metrics and all tests for the object names - # NOTE Probably there is a more elegant way?! - test_names_known = np.unique(self.test_names) - metric_names_known = np.unique(self.metric_names) - object_names_known = np.unique(self.object_names) - - object_names_to_add = [] - metric_names_to_add = [] - test_names_to_add = [] - - for object_name, metric_name in product(object_names_known, metric_names_known): - mask = (self.object_names == object_name) & (self.metric_names == metric_name) - if not np.any(mask): - object_names_to_add.extend([object_name] * len(test_names_known)) - metric_names_to_add.extend([metric_name] * len(test_names_known)) - test_names_to_add.extend(test_names_known) - continue - present_test_names = self.test_names[mask] - test_names_not_present = test_names_known[~np.isin(present_test_names, test_names_known)] - test_names_to_add.extend(test_names_not_present) - metric_names_to_add.extend([metric_name] * len(test_names_not_present)) - object_names_to_add.extend([object_name] * len(test_names_not_present)) - - self.object_names = np.array(np.append(self.object_names, object_names_to_add)) - self.metric_names = np.array(np.append(self.metric_names, metric_names_to_add)) - self.test_names = np.array(np.append(self.test_names, test_names_to_add)) - self.tests = np.array(np.append(self.tests, [TestLimits(tnta) for tnta in test_names_to_add])) - self.mask_any = np.full(self.test_names.shape, True) def test(self, metrics): @@ -253,42 +214,51 @@ class RelVal: def __init__(self): # metric names that should be considered (if empty, all) self.include_metrics = [] + # metric names that should be excluded, takes precedence over self.include_metrics self.exclude_metrics = [] - # lists of regex to include/exclude objects by name + # lists of regex to include/exclude objects by their names self.include_patterns = None self.exclude_patterns = None - # collecting everything we have; all of the following will have the same length in the end + # collecting everything we have; the following three members will have the same length self.object_names = None self.metric_names = None # metric objects self.metrics = None - # object and metric names known to this RelVal + # unique object and metric names self.known_objects = None self.known_metrics = None - # collecting all results; all of the following will have the same length in the end + # collecting all results; the following three members will have the same length self.results = None - # indices to refer to self.object_names, self.metric_names and self.metrics + # each index refers to the corresponding object in self.object_names, self.metric_names and self.metrics self.results_to_metrics_idx = None + + # unique list of test names self.known_test_names = None # to store some annotations self.annotations = None - def enable_metrics(self, metrics): - if not metrics: + def enable_metrics(self, metric_names): + """ + Enable a list of metrics given their names + """ + if not metric_names: return - for metric in metrics: + for metric in metric_names: if metric in self.include_metrics: continue self.include_metrics.append(metric) - def disable_metrics(self, metrics): - if not metrics: + def disable_metrics(self, metric_names): + """ + Disable a list of metrics given their names + """ + if not metric_names: return - for metric in metrics: + for metric in metric_names: if metric in self.exclude_metrics: continue self.exclude_metrics.append(metric) @@ -367,29 +337,25 @@ def add_result(self, metric_idx, result): self.results_to_metrics_idx.append(metric_idx) self.results.append(result) - def load(self, summaries_to_test): + def get_metric_checking_dict(self, in_dict): + """ + Check if that metric is already known + """ + if self.metrics is None: + return None, None - self.annotations = [] - self.object_names = [] - self.metric_names = [] - self.metrics = [] - self.results_to_metrics_idx = [] - self.results = [] + metric = Metric(in_dict=in_dict) - for summary_to_test in summaries_to_test: - summary_to_test = self.read(summary_to_test) - if annotations := summary_to_test.get(RelVal.KEY_ANNOTATIONS, None): - self.annotations.append(annotations) - for line in summary_to_test[RelVal.KEY_OBJECTS]: - metric = Metric(in_dict=line) - if not self.add_metric(metric): - continue + for idx, search_metric in enumerate(self.metrics): + if metric == search_metric: + return idx, search_metric - if "result_name" in line: - # NOTE We could think about not duplicating metrics. - # Because there is the same metric for each of the corresponding test results - self.add_result(len(self.metrics) - 1, Result(in_dict=line)) + return None, metric + def to_numpy(self): + """ + Convert everything that is a list to numpy for faster querying later on + """ self.known_objects = np.unique(self.object_names) self.known_metrics = np.unique(self.metric_names) @@ -402,8 +368,42 @@ def load(self, summaries_to_test): self.results_to_metrics_idx = np.array(self.results_to_metrics_idx, dtype=int) if self.results else None self.test_names_results = np.array([r.name for r in self.results]) if self.results else None self.known_test_names = np.unique(self.test_names_results) if self.results else None - self.result_filter_mask = np.full(self.known_test_names.shape, True) if self.results else None self.results = np.array(self.results, dtype=Result) if self.results else None + self.result_filter_mask = np.full(self.results.shape, True) if self.results is not None else None + + def load(self, summaries_to_test): + """ + Loads and populates this object from a dictionary + """ + self.annotations = [] + self.object_names = [] + self.metric_names = [] + self.metrics = [] + self.results_to_metrics_idx = [] + self.results = [] + + for summary_to_test in summaries_to_test: + # loop over the list of dictionaries given + summary_to_test = self.read(summary_to_test) + if annotations := summary_to_test.get(RelVal.KEY_ANNOTATIONS, None): + self.annotations.append(annotations) + for line in summary_to_test[RelVal.KEY_OBJECTS]: + # each list object corresponds to and object with a certain test result + # first of all we check if that metric is already loaded + idx, metric = self.get_metric_checking_dict(line) + if idx is None: + # in this case, this metric is new + idx = len(self.metrics) - 1 + if not self.add_metric(metric): + # only attempt to add if that metric is not yet there + continue + + if "result_name" in line: + # add this result; the result will be mapped to the metric it is based on via the index + self.add_result(idx, Result(in_dict=line)) + + # convert everything that was a list before to numpy objects + self.to_numpy() def get_metrics(self, object_name=None, metric_name=None): """ @@ -422,7 +422,7 @@ def get_metrics(self, object_name=None, metric_name=None): mask = mask & (self.any_mask if metric_name is None else np.isin(self.metric_names, metric_name)) return self.object_names[mask], self.metric_names[mask], self.metrics[mask] - def apply(self, evaluator): + def apply_evaluator(self, evaluator): """ Apply loaded tests """ @@ -438,20 +438,32 @@ def apply(self, evaluator): self.results_to_metrics_idx, self.results = evaluator.test(self.metrics) self.test_names_results = np.array([r.name for r in self.results]) self.known_test_names = np.unique(self.test_names_results) - self.result_filter_mask = np.full(self.known_test_names.shape, True) + self.result_filter_mask = np.full(self.results.shape, True) def interpret(self, interpret_func): + """ + Add an interpretation to the Result objects based on a function given by the user + """ for metric_idx, result in zip(self.results_to_metrics_idx, self.results): interpret_func(result, self.metrics[metric_idx]) def filter_results(self, filter_func): + """ + Construct a mask to filter results without losing any of them + """ if self.results is None: return self.result_filter_mask = [filter_func(result) for result in self.results] def query_results(self, query_func=None): + """ + Query Result objects based on a function given by the user + + Return matching Result objects along with names + """ mask = np.array([query_func is None or query_func(result) for result in enumerate(self.results)]) - mask = mask & self.result_filter_mask + if self.result_filter_mask is not None: + mask = mask & self.result_filter_mask idx = self.results_to_metrics_idx[mask] return np.take(self.object_names, idx), np.take(self.metric_names, idx), self.test_names_results[idx], self.results[idx] @@ -474,15 +486,28 @@ def get_metric_name(self, idx): return self.known_metrics[idx] def get_result_per_metric_and_test(self, metric_index_or_name=None, test_index_or_name=None): + """ + Return Result objects that belong to given metric or test + """ test_name = test_index_or_name if (isinstance(test_index_or_name, str) or test_index_or_name is None) else self.known_test_names[test_index_or_name] metric_name = metric_index_or_name if (isinstance(metric_index_or_name, str) or metric_index_or_name is None) else self.known_metrics[metric_index_or_name] metric_idx = np.argwhere(self.metric_names == metric_name) if metric_name is not None else self.results_to_metrics_idx - mask = np.isin(self.results_to_metrics_idx, metric_idx) & self.result_filter_mask + mask = np.isin(self.results_to_metrics_idx, metric_idx) + if self.result_filter_mask is not None: + mask = mask & self.result_filter_mask if test_name is not None: mask = mask & (self.test_names_results == test_name) return np.take(self.object_names, self.results_to_metrics_idx[mask]), self.results[mask] def get_result_matrix_objects_metrics(self, test_index): + """ + Return a matrix of Result objects + + vertical axis: object names + horizontal axis: metric names + + in addition return metric and object names so the user knows what she gets + """ mask = self.test_names_results == (self.known_test_names[test_index]) idx = self.results_to_metrics_idx[mask] results = self.results[mask] @@ -497,9 +522,15 @@ def get_result_matrix_objects_metrics(self, test_index): return metric_names, object_names, np.reshape(results[idx], (len(object_names), len(metric_names))) def yield_metrics_results_per_object(self): + """ + One-by-one return metrics and results of objects + """ results = None if self.results is not None: - mask = self.result_filter_mask + if self.result_filter_mask is not None: + mask = self.result_filter_mask + else: + mask = np.full(self.results.shape, True) idx = self.results_to_metrics_idx[mask] object_names = np.take(self.object_names, idx) metrics = np.take(self.metrics, idx) @@ -515,10 +546,13 @@ def yield_metrics_results_per_object(self): yield object_name, yield_metrics, yield_results def write(self, filepath, annotations=None): + """ + Write everything to a JSON file + Structure corresponds to what ROOT's RelVal returns so in turn it can be used to construct a RelVal object again + """ all_objects = [] - # TODO return one flat dictionary not a nested one def make_dict_include_results(object_name, metric, result): return {RelVal.KEY_OBJECT_NAME: object_name} | metric.as_dict() | result.as_dict() @@ -528,7 +562,7 @@ def make_dict_exclude_results(object_name, metric, *args): if self.results is None: object_names = self.object_names metrics = self.metrics - results = np.empty(metric.shape, dtype=bool) + results = np.empty(metrics.shape, dtype=bool) make_dict = make_dict_exclude_results else: object_names = np.take(self.object_names, self.results_to_metrics_idx) @@ -546,20 +580,26 @@ def make_dict_exclude_results(object_name, metric, *args): json.dump(final_dict, f, indent=2) -def get_summaries_or_from_file(in_objects): - - if len(in_objects) == 1 and in_objects[0].startswith("@"): - with open(in_objects[0][1:], "r") as f: +def get_paths_or_from_file(paths): + """ + Either simply return the paths or extract them from a text file + """ + if len(paths) == 1 and paths[0].startswith("@"): + with open(paths[0][1:], "r") as f: return f.read().splitlines() - return in_objects + return paths def initialise_thresholds(evaluator, rel_val, rel_val_thresholds, thresholds_default, thresholds_margin, thresholds_combine="mean"): - + """ + Add thresholds to the Evaluator as one test case + """ # The default thresholds will be derived and set for all the objects and metrics that we find in the RelVal to test _, _, metrics = rel_val.get_metrics() for metric in metrics: + # get the default thresholds for each metric proposed_threshold = thresholds_default.get(metric.name, metric.proposed_threshold) if thresholds_default else metric.proposed_threshold + # depending on what's better (lower/greater), set the std boundaries std = (None, 0) if metric.lower_is_better else (0, None) evaluator.add_limits(metric.object_name, metric.name, TestLimits("threshold_default", proposed_threshold, std)) @@ -569,56 +609,74 @@ def initialise_thresholds(evaluator, rel_val, rel_val_thresholds, thresholds_def for object_name in rel_val_thresholds.known_objects: for metric_name in rel_val_thresholds.known_metrics: + # get metric for given objects by name _, _, metrics = rel_val_thresholds.get_metrics((object_name,), (metric_name,)) + if not np.any(metrics): continue + # collect all values from all metrics for this object values = [m.value for m in metrics if m.comparable] + # check what is better, lower or greater lower_is_better = metrics[0].lower_is_better factor = 1 if lower_is_better else -1 if not values: + evaluator.add_limits(object_name, metric_name, TestLimits("threshold_user")) continue if thresholds_combine == "mean": + # combine the values, by default take the mean as the threshold mean_central = np.mean(values) else: + # otherwise take the extremum mean_central = factor * max([factor * v for v in values]) + margin = thresholds_margin[metric_name] * mean_central if thresholds_margin and metric_name in thresholds_margin else 0 + # put together the std limits and add the TestLimits to the Evaluator if factor > 0: low = None - up = (1 + thresholds_margin[metric_name]) * mean_central + up = margin else: up = None - low = (1 - thresholds_margin) * mean_central + low = margin evaluator.add_limits(object_name, metric_name, TestLimits("threshold_user", mean_central, (low, up))) -def initialise_regions(evaluator, regions): - rel_val_regions = RelVal() - rel_val_regions.load(regions) +def initialise_regions(evaluator, rel_val_regions): + """ + Add regions to the Evaluator as test case + """ + # Loop through everything for object_name in rel_val_regions.known_objects: for metric_name in rel_val_regions.known_metrics: _, _, metrics = rel_val_regions.get_metrics((object_name,), (metric_name,)) + # get all the metric values for the given object and a particular metric values = [m.value for m in metrics if m.comparable] + # extract some properties of the metrics that need to be known proposed_threshold = metrics[0].proposed_threshold lower_is_better = metrics[0].lower_is_better + # a list of metric values where outliers are removed values_central = [] + # a list of metric values with only outliers values_outlier = [] for v in values: diff = v - proposed_threshold if (diff < 0 and lower_is_better) or (diff > 0 and not lower_is_better): - # if the value is below and lower is better (or the other way round), then accept it + # if the value is below and lower is better (or the other way round), then accept it because it is definitely better than even the proposed threshold values_central.append(v) continue if diff != 0: + # check how far off the calculated difference is from the proposed value diff = abs(proposed_threshold / diff) if diff < 0.1: - # this means we accept up to an order of magnitude + # this means we accept up to an order of magnitude, this is hence an outlier values_outlier.append(v) continue + # if this is reached, the value is worse than the proposed threshold but only by less than one order of magnitude values_central.append(v) + # now get the means of this region with their std for both central and outliers mean_central = np.mean(values_central) std_central = np.std(values_central) if np.any(values_outlier): @@ -627,16 +685,22 @@ def initialise_regions(evaluator, regions): else: mean_outlier = None std_outlier = None + # add these mean and std values as two different test limits evaluator.add_limits(object_name, metric_name, TestLimits("regions_tight", mean_central, (std_central, std_central))) evaluator.add_limits(object_name, metric_name, TestLimits("regions_loose", mean_outlier, (std_outlier, std_outlier))) def run_macro(cmd, log_file, cwd=None): + """ + Wrapper to run a command line + """ p = Popen(split(cmd), cwd=cwd, stdout=PIPE, stderr=STDOUT, universal_newlines=True) + # open a logfile and write to it line by line log_file = open(log_file, 'a') for line in p.stdout: log_file.write(line) p.wait() + # when done, close the logfile and return the cmd's return code log_file.close() return p.returncode @@ -669,9 +733,54 @@ def print_summary(rel_val, interpretations, long=False): def get_summary_path(path): + """ + Get the full path to Summary.json + + If a directory is given, look for the file inside + """ if isdir(path): path = join(path, "Summary.json") if exists(path): return path print(f"ERROR: Cannot neither find {path}.") return None + + +def copy_overlays(rel_val, input_dir, output_dir): + """ + copy overlay plots in this summary from the input directory to the output directory + """ + input_dir = abspath(input_dir) + output_dir = abspath(output_dir) + + if not exists(input_dir): + print(f"ERROR: Input directory {input_dir} does not exist") + return 1 + + in_out_same = input_dir == output_dir + + input_dir_new = input_dir + "_tmp" + if in_out_same: + # move input directory + rename(input_dir, input_dir_new) + input_dir = input_dir_new + + if not exists(output_dir): + makedirs(output_dir) + + object_names, _ = rel_val.get_result_per_metric_and_test() + object_names = list(set(object_names)) + + ret = 0 + for object_name in object_names: + filename=join(input_dir, f"{object_name}.png") + if exists(filename): + copy(filename, output_dir) + else: + print(f"File {filename} not found.") + ret = 1 + + if in_out_same: + rmtree(input_dir) + + return ret From 7732510ecf82fb0b42012b9ca1cb9ccfee793cd5 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 5 Feb 2024 12:24:44 +0100 Subject: [PATCH 0340/1239] [SimWF] Comments for anchored WF script (#1437) * remove `currenttime` and use `timestamp` everywhere * add comments in determine_timestamp, especially making clear where seconds, milliseconds or microseconds are used --- MC/bin/o2dpg_sim_workflow_anchored.py | 66 +++++++++++++++++---------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 2feba79ce..1876e4877 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -259,33 +259,56 @@ def determine_timestamp(sor, eor, splitinfo, cycle, ntf, HBF_per_timeframe = 256 Determines the timestamp and production offset variable based on the global properties of the production (MC split, etc) and the properties of the run. ntf is the number of timeframes per MC job + + Args: + sor: int + start-of-run in milliseconds since epoch + eor: int + end-of-run in milliseconds since epoch + splitinfo: tuple (int, int) + splitinfo[0]: split ID of this job + splitinfo[1]: total number of jobs to split into + cycle: int + cycle of this productions. Typically a run is not entirely filled by and anchored simulation + but only a proportion of events is simulated. + With increasing number of cycles, the data run is covered more and more. + ntf: int + number of timeframes + HBF_per_timeframe: int + number of orbits per timeframe + Returns: + int: timestamp in milliseconds + int: production offset aka "which timeslot in this production to simulate" """ totaljobs = splitinfo[1] thisjobID = splitinfo[0] - print (f"Start-of-run : {sor}") - print (f"End-of-run : {eor}") - time_length_inmus = 1000*(eor - sor) # time length in micro seconds - timestamp_delta = time_length_inmus / totaljobs + # length of this run in micro seconds, since we use the orbit duration in micro seconds + time_length_inmus = 1000 * (eor - sor) + # figure out how many timeframes fit into this run range + # take the number of orbits per timeframe and multiply by orbit duration to calculate how many timeframes fit into this run ntimeframes = time_length_inmus / (HBF_per_timeframe * LHCOrbitMUS) - norbits = time_length_inmus / LHCOrbitMUS + # also calculate how many orbits fit into the run range print (f"This run has space for {ntimeframes} timeframes") - print (f"This run has {norbits} orbits") - # ntimeframes is the total number of timeframes possible - # if we have totaljobs number of jobs - maxtimeframesperjob = ntimeframes // totaljobs - orbitsperjob = norbits // totaljobs + # figure out how many timeframes can maximally be covered by one job + maxtimeframesperjob = ntimeframes / totaljobs print (f"Each job can do {maxtimeframesperjob} maximally at a prod split of {totaljobs}") - print (f"With each job doing {ntf} timeframes, this corresponds to a filling rate of ", ntf/maxtimeframesperjob) + print (f"With each job doing {ntf} timeframes, this corresponds to a filling rate of {ntf / maxtimeframesperjob}") # filling rate should be smaller than 100% assert(ntf <= maxtimeframesperjob) - maxcycles = maxtimeframesperjob // ntf - print (f"We can do this amount of cycle iterations to achieve 100%: ", maxcycles) + # each cycle populates more and more run range. The maximum number of cycles to populate the run fully is: + maxcycles = maxtimeframesperjob / ntf + print (f"We can do this amount of cycle iterations to achieve 100%: {maxcycles}") + # overall, we have maxcycles * totaljobs slots to fill the run range with ntf timeframes per slot + # figure out in which slot to simulate production_offset = int(thisjobID * maxcycles) + cycle + # add the time difference of this slot to start-of-run to get the final timestamp timestamp_of_production = sor + production_offset * ntf * HBF_per_timeframe * LHCOrbitMUS / 1000 + # this is a closure test. If we had prefect floating point precision everywhere, it wouldn't fail. + # But since we don't have that and there are some int casts as well, better check again. assert (timestamp_of_production >= sor) assert (timestamp_of_production <= eor) return int(timestamp_of_production), production_offset @@ -323,16 +346,17 @@ def main(): exit (1) first_orbit = ctp_scalers.getOrbitLimit().first + # SOR and EOR values in milliseconds sor = ctp_scalers.getTimeLimit().first eor = ctp_scalers.getTimeLimit().second if args.use_rct_info: first_orbit = GLOparams["FirstOrbit"] + # SOR and EOR values in milliseconds sor = GLOparams["SOR"] eor = GLOparams["EOR"] - # determine timestamp, and production offset for the final - # MC job to run + # determine timestamp, and production offset for the final MC job to run timestamp, prod_offset = determine_timestamp(sor, eor, [args.split_id - 1, args.prod_split], args.cycle, args.tf, GLOparams["OrbitsPerTF"]) # this is anchored to @@ -341,13 +365,8 @@ def main(): print ("Determined timestamp to be : ", timestamp) print ("Determined offset to be : ", prod_offset) - currentorbit = first_orbit + prod_offset * GLOparams["OrbitsPerTF"] # orbit number at production start - currenttime = sor + prod_offset * GLOparams["OrbitsPerTF"] * LHCOrbitMUS // 1000 # timestamp in milliseconds - - print ("Production put at time : " + str(currenttime)) - # retrieve the GRPHCIF object - grplhcif = retrieve_GRPLHCIF(ccdbreader, int(currenttime)) + grplhcif = retrieve_GRPLHCIF(ccdbreader, int(timestamp)) eCM = grplhcif.getSqrtS() A1 = grplhcif.getAtomicNumberB1() A2 = grplhcif.getAtomicNumberB2() @@ -385,8 +404,9 @@ def main(): effTrigger = 28.0 # this is ZDC else: effTrigger = 0.759 - - rate = retrieve_MinBias_CTPScaler_Rate(ctp_scalers, currenttime/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) + + # time needs to be converted to seconds ==> timestamp / 1000 + rate = retrieve_MinBias_CTPScaler_Rate(ctp_scalers, timestamp/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) if rate != None: # if the rate calculation was successful we will use it, otherwise we fall back to some rate given as part From ead8c0ba8d2161a0c50d1829e020133b699d5a53 Mon Sep 17 00:00:00 2001 From: shahoian Date: Sun, 4 Feb 2024 03:16:03 +0100 Subject: [PATCH 0341/1239] Adjust TPC scaling options to O2 PR12653 Since the CTP lumi is requested by default even if scaling with IDCs is used, the parameters to configure TPC corrections TPC_CORR_SCALING are modidied as: ALIEN_JDL_MEANIRFORTPC (by default: empty) >0 : use at face value to set TPCCorrMap.lumiMean, overriding mean lumi of the map <0 : disable ALL corrections ALIEN_JDL_INSTIRFORTPC (by default: CTP) >0 : use at face value to set TPCCorrMap.lumiInst which will override ONLY CTP lumi value. (which does not affect the lumi used for scaling with IDCs) CTP: use Lumi from the CTP Data (as above: not necessarilly for corrections scaling) CTPCCDB: extract CTP lumi from CCDB CTP scalers and override CTP Lumi TPCCorrMap.lumiInst (same comment as above) New variable defining what is used for corrections scaling was added ALIEN_JDL_TPCSCALINGSOURCE (by default set to CTP) NO_SCALING : no scaling applied ( --lumi-type is set to 0) CTP: inst lumi is used also for scaling ( --lumi-type 1) IDCCCDB: use for scaling the TPC scaler from IDCs. --- .../configurations/asyncReco/setenv_extra.sh | 127 +++++++++++------- 1 file changed, 75 insertions(+), 52 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 4c95be58a..adbdbaaec 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -296,87 +296,110 @@ elif [[ $ALIGNLEVEL == 1 ]]; then CUT_MATCH_CHI2=160 export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=5;tpcitsMatch.crudeNSigma2Cut[0]=100;tpcitsMatch.crudeNSigma2Cut[1]=100;tpcitsMatch.crudeNSigma2Cut[2]=100;tpcitsMatch.crudeNSigma2Cut[3]=100;tpcitsMatch.crudeNSigma2Cut[4]=100;" - # enabling TPC calibration scaling + #-------------------------------------- TPC corrections ----------------------------------------------- + # we need to provide to TPC + # 1) interaction rate info (lumi) used for scaling or errors and possible of the corrections : INST_IR_FOR_TPC + # 2) what to use for corrections scaling (lumi or IDC scalers or no scaling at all) : TPC_SCALING_SOURCE # the default is to use CTP, unless specified differently in the JDL... INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-CTP} - #...but for 2022 data, where we will rely on different settings depending on the period; note that if ALIEN_JDL_INSTIRFORTPC is set, it has precedence + TPC_SCALING_SOURCE=${ALIEN_JDL_TPCSCALINGSOURCE-CTP} + # MEAN_IR_FOR_TPC allows (1) to alter the map mean IR if >0 or (2) disable all corrections if <0 + MEAN_IR_FOR_TPC=${ALIEN_JDL_MEANIRFORTPC-} + if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then - INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-CTPCCDB} + INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-CTPCCDB} # by default override inst.IR by the mean IR from CCDB and use it for scaling fi if [[ $PERIOD == "LHC22s" ]]; then - INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-0} # in this way, only TPC/Calib/CorrectionMaps is applied, and we know that for 22s it is the same as TPC/Calib/CorrectionMapsRef; note that if ALIEN_JDL_INSTIRFORTPC is set, it has precedence + TPC_SCALING_SOURCE=${ALIEN_JDL_TPCSCALINGSOURCE-NO_SCALING} # in this way, only TPC/Calib/CorrectionMaps is applied, and we know that for 22s it is the same as TPC/Calib/CorrectionMapsRef; elif [[ $PERIOD == @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]]; then INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-1} # scaling with very small value for low IR fi - # in MC, we set it to a negative value to disable completely the corrections (not yet operational though, please check O2); + # in MC, we disable completely the corrections # note that if ALIEN_JDL_INSTIRFORTPC is set, it has precedence if [[ $ALIEN_JDL_LPMPRODUCTIONTYPE == "MC" ]] && [[ $O2DPG_ENABLE_TPC_DISTORTIONS != "ON" ]]; then - INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC--1} + MEAN_IR_FOR_TPC=${ALIEN_JDL_MEANIRFORTPC--1} fi - # now we set the options + DISABLE_CORRECTIONS= [[ -n "$ALIEN_JDL_MSHAPE_CORRECTION" && $ALIEN_JDL_MSHAPE_CORRECTION == "0" ]] && ENABLE_MSHAPE=0 || ENABLE_MSHAPE=1 - if [[ $INST_IR_FOR_TPC -gt 0 ]]; then # externally imposed IR for scaling - echo "Applying externally provided IR for scaling, $INST_IR_FOR_TPC Hz" - export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" - elif [[ $INST_IR_FOR_TPC == 0 ]]; then # when zero, only the TPC/Calib/CorrectionMaps is applied - echo "Passed valued for scaling is zero, only TPC/Calib/CorrectionMaps will be applied" - export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" - elif [[ $INST_IR_FOR_TPC -lt 0 ]]; then # do not apply any correction - echo "Passed valued for scaling is smaller than zero, no scaling will be applied" - echo "NOTA BENE: In the future, this value will signal to not apply any correction at all, which is not operational yet (but please check, as it depends on O2)" - ENABLE_MSHAPE=0 + if [[ -n $MEAN_IR_FOR_TPC ]] ; then # firs check if corrections were not disabled via MEAN_IR_FOR_TPC + if [[ $MEAN_IR_FOR_TPC -gt 0 ]] ; then # positive value overrides map mean lumi + echo "Applying externally provided map mean IR for scaling, $MEAN_IR_FOR_TPC Hz" + export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$MEAN_IR_FOR_TPC;" # take mean lumy at face value + elif [[ $MEAN_IR_FOR_TPC -lt 0 ]] ; then # negative mean lumi disables all corrections + echo "Negative MEAN_IR_FOR_TPC -> all TPC corrections will be ignored" + export TPC_CORR_SCALING+=" --lumi-type 0 " + export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$MEAN_IR_FOR_TPC;" + ENABLE_MSHAPE=0 + DISABLE_CORRECTIONS=1 + else + echo "Did not recognize MEAN_IR_FOR_TPC = $MEAN_IR_FOR_TPC" + return 1 + fi + fi # MEAN_IR_FOR_TPC overridden + + # set IR for TPC, even if it is not used for corrections scaling + if [[ $INST_IR_FOR_TPC -gt 0 ]]; then # externally imposed CTP IR + echo "Applying externally provided istantaneous IR $INST_IR_FOR_TPC Hz" export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$INST_IR_FOR_TPC" - elif [[ $INST_IR_FOR_TPC == "CTPCCDB" ]]; then # using what we have in the CCDB CTP counters, extracted at the beginning of the script - echo "Using CTP CCDB which gave the mean IR of the run at the beginning of the script ($RUN_IR Hz)" - export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$RUN_IR" elif [[ $INST_IR_FOR_TPC == "CTP" ]]; then if ! has_detector CTP ; then - echo "TPC correction with CTP Lumi is requested but CTP is not in the WORKFLOW_DETECTORS=$WORKFLOW_DETECTORS" + echo "CTP Lumi is for TPC corrections but CTP is not in the WORKFLOW_DETECTORS=$WORKFLOW_DETECTORS" return 1 fi echo "Using CTP inst lumi stored in data" - export TPC_CORR_SCALING+=" --lumi-type 1 " - elif [[ $INST_IR_FOR_TPC == "IDCCCDB" ]]; then - echo "TPC correction with IDC from CCDB will be used" - export TPC_CORR_SCALING+=" --lumi-type 2 " - if [[ $ALIEN_JDL_USEDERIVATIVESFORSCALING == "1" ]]; then - export TPC_CORR_SCALING+=" --corrmap-lumi-mode 1 " - fi - else - echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" + elif [[ $INST_IR_FOR_TPC == "CTPCCDB" ]]; then # using what we have in the CCDB CTP counters, extracted at the beginning of the script + echo "Using CTP CCDB which gave the mean IR of the run at the beginning of the script ($RUN_IR Hz)" + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$RUN_IR" + else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" return 1 fi - if [[ $ENABLE_MSHAPE == "1" ]]; then - export TPC_CORR_SCALING+=" --enable-M-shape-correction " + # now set the source of the corrections + if [[ $DISABLE_CORRECTIONS != 1 ]] ; then + if [[ $TPC_SCALING_SOURCE == "NO_SCALING" ]]; then + echo "NO SCALING is requested: only TPC/Calib/CorrectionMapsV2... will be applied" + export TPC_CORR_SCALING+=" --lumi-type 0 " + elif [[ $TPC_SCALING_SOURCE == "CTP" ]]; then + echo "CTP Lumi from data will be used for TPC scaling" + export TPC_CORR_SCALING+=" --lumi-type 1 " + elif [[ $TPC_SCALING_SOURCE == "IDCCCDB" ]]; then + echo "TPC correction with IDC from CCDB will be used" + export TPC_CORR_SCALING+=" --lumi-type 2 " + if [[ $ALIEN_JDL_USEDERIVATIVESFORSCALING == "1" ]]; then + export TPC_CORR_SCALING+=" --corrmap-lumi-mode 1 " + fi + else + echo "Unknown setting for TPC_SCALING_SOURCE = $TPC_SCALING_SOURCE (with ALIEN_JDL_TPCSCALINGSOURCE = $ALIEN_JDL_TPCSCALINGSOURCE)" + fi + fi + + if ! has_detector CTP ; then + echo "CTP is not in the list of detectors, disabling CTP Lumi input request" + export TPC_CORR_SCALING+=" --disable-ctp-lumi-request " fi - if [[ -n $ALIEN_JDL_MEANIRFORTPC && $ALIEN_JDL_MEANIRFORTPC > 0 ]]; then # externally imposed TPC map mean IR for scaling - export TPC_CORR_SCALING+=";TPCCorrMap.lumiMean=$ALIEN_JDL_MEANIRFORTPC" + if [[ $ENABLE_MSHAPE == "1" ]]; then + export TPC_CORR_SCALING+=" --enable-M-shape-correction " fi - - if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]] && ([[ -z $INST_IR_FOR_TPC ]] || [[ $INST_IR_FOR_TPC == "CTP" ]]); then - echo "We are in PbPb 2023, the default - for now - is to use CTP in the data" - unset TPC_CORR_SCALING - export TPC_CORR_SCALING=";TPCCorrMap.lumiInstFactor=2.414;TPCCorrMap.lumiMean=0 --lumi-type 1 " - if [[ $SCALE_WITH_ZDC == 0 ]]; then - # scaling with FT0 - if [[ $SCALE_WITH_FT0 == 1 ]]; then - export TPC_CORR_SCALING=" --ctp-lumi-source 1 --lumi-type 1 TPCCorrMap.lumiInstFactor=135.;TPCCorrMap.lumiMean=0" - else - echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." - return 1 - fi - fi - if [[ $ENABLE_MSHAPE == "1" ]]; then - export TPC_CORR_SCALING+=" --enable-M-shape-correction " + + if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]] ; then + if [[ $SCALE_WITH_ZDC == 1 ]]; then + echo "For 2023 PbPb ZDC inst. lumi applying factor 2.414" + export TPC_CORR_SCALING+=";TPCCorrMap.lumiInstFactor=2.414;" + elif [[ $SCALE_WITH_FT0 == 1 ]]; then + echo "For 2023 PbPb FT0 inst. lumi applying factor 135." + export TPC_CORR_SCALING+="TPCCorrMap.lumiInstFactor=135.;" + else + echo "Neither ZDC nor FT0 are in the run, and this is from 2023 PbPb: we cannot scale TPC ditortion corrections, aborting..." + return 1 fi fi echo "Final setting for TPC scaling is:" echo $TPC_CORR_SCALING + #-------------------------------------- TPC corrections (end)-------------------------------------------- if [[ $PERIOD != @(LHC22c|LHC22d|LHC22e|JUN|LHC22f) ]] ; then echo "Setting TPCCLUSTERTIMESHIFT to 0" @@ -385,8 +408,8 @@ elif [[ $ALIGNLEVEL == 1 ]]; then echo "We are in period $PERIOD, we need to keep the correction for the TPC cluster time, since no new vdrift was extracted" fi - TRACKTUNETPCINNER="trackTuneParams.tpcCovInnerType=1;trackTuneParams.tpcCovInner[0]=0.01;trackTuneParams.tpcCovInner[1]=1.;trackTuneParams.tpcCovInner[2]=4e-7;trackTuneParams.tpcCovInner[3]=4.e-5;trackTuneParams.tpcCovInner[4]=6.8e-6;" - TRACKTUNETPCOUTER="trackTuneParams.tpcCovOuterType=1;trackTuneParams.tpcCovOuter[0]=0.01;trackTuneParams.tpcCovOuter[1]=1.;trackTuneParams.tpcCovOuter[2]=4e-7;trackTuneParams.tpcCovOuter[3]=4.e-5;trackTuneParams.tpcCovOuter[4]=6.8e-6;" + TRACKTUNETPCINNER="trackTuneParams.tpcCovInnerType=1;trackTuneParams.tpcCovInner[0]=0.1;trackTuneParams.tpcCovInner[1]=1.;trackTuneParams.tpcCovInner[2]=6.3e-4;trackTuneParams.tpcCovInner[3]=6.3e-3;trackTuneParams.tpcCovInner[4]=2.6e-3;" + TRACKTUNETPCOUTER="trackTuneParams.tpcCovOuterType=1;trackTuneParams.tpcCovOuter[0]=0.1;trackTuneParams.tpcCovOuter[1]=1.;trackTuneParams.tpcCovOuter[2]=6.3e-4;trackTuneParams.tpcCovOuter[3]=6.3e-3;trackTuneParams.tpcCovOuter[4]=2.6e-3;" fi From cd07a5c91c559d20481093d7769ec82478a639c3 Mon Sep 17 00:00:00 2001 From: swenzel Date: Fri, 2 Feb 2024 13:08:02 +0100 Subject: [PATCH 0342/1239] prototypic addition of TPC time series O2-4612 https://its.cern.ch/jira/browse/O2-4612 --- MC/bin/o2dpg_sim_workflow.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 70dcafb71..5f116bab5 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1421,9 +1421,23 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # Enable CTP readout replay for triggered detectors (EMCAL, HMPID, PHOS/CPV, TRD) # Needed untill triggers are supported in CTP simulation AODtask['cmd'] += ' --ctpreadout-create 1' - workflow['stages'].append(AODtask) + # TPC - time-series objects + # initial implementation taken from comments in https://its.cern.ch/jira/browse/O2-4612 + # TODO: this needs to be made configurable (as a function of which detectors are actually present) + tpctsneeds = [ TPCRECOtask['name'], + ITSTPCMATCHtask['name'], + TOFTPCMATCHERtask['name'], + PVFINDERtask['name'] + ] + TPCTStask = createTask(name='tpctimeseries_'+str(tf), needs=tpctsneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='2000', cpu='1') + TPCTStask['cmd'] = 'o2-global-track-cluster-reader --disable-mc --cluster-types "TOF" --track-types "ITS,TPC,ITS-TPC,ITS-TPC-TOF,ITS-TPC-TRD-TOF"' + TPCTStask['cmd'] += ' --primary-vertices ' + TPCTStask['cmd'] += ' | o2-tpc-time-series-workflow --enable-unbinned-root-output --sample-unbinned-tsallis --sampling-factor 0.1 ' + TPCTStask['cmd'] += putConfigValuesNew() + ' ' + getDPL_global_options(bigshm=True) + workflow['stages'].append(TPCTStask) + # AOD merging / combination step (as individual stages) --> for the moment deactivated in favor or more stable global merging """ aodmergerneeds = [ AODtask['name'] ] From e568993c65a8c71ba99afd8df0711835a7a73251 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 2 Feb 2024 13:15:59 +0100 Subject: [PATCH 0343/1239] Bug fix for sampling --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index adbdbaaec..bc0797b71 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -613,7 +613,7 @@ if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority export SAMPLINGFACTORTIMESERIES=${ALIEN_JDL_SAMPLINGFACTORTIMESERIES} fi - export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${ALIEN_JDL_SAMPLINGFACTORTIMESERIES}" + export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${SAMPLINGFACTORTIMESERIES}" fi fi From 232fdbde236da69d9f2240bbc6b1324dc2a99761 Mon Sep 17 00:00:00 2001 From: ValerieRamillien <95756671+ValerieRamillien@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:04:11 +0100 Subject: [PATCH 0344/1239] Update mid.json proposed configuration changes --- DATA/production/qc-async/mid.json | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/DATA/production/qc-async/mid.json b/DATA/production/qc-async/mid.json index 9c6d6997e..7502d9f30 100644 --- a/DATA/production/qc-async/mid.json +++ b/DATA/production/qc-async/mid.json @@ -3,23 +3,18 @@ "config": { "database": { "implementation": "CCDB", - "host": "ali-qcdb.cern.ch:8083", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" + "host": "ccdb-test.cern.ch:8080" }, "Activity": { - "number": "42", - "type": "2" }, "monitoring": { - "url": "influxdb-unix:///tmp/telegraf.sock" + "url": "infologger:///debug?qc" }, "consul": { - "url": "null:8500" + "url": "" }, "conditionDB": { - "url": "null:8083" + "url": "ccdb-test.cern.ch:8080" } }, "tasks": { @@ -30,11 +25,9 @@ "moduleName": "QcMID", "detectorName": "MID", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "digits:MID/DATA;digits_rof:MID/DATAROF", - "query_comment": "100% sampling" + "query": "digits:MID/DATA;digits_rof:MID/DATAROF" } }, "MIDClusters": { @@ -44,11 +37,9 @@ "moduleName": "QcMID", "detectorName": "MID", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS", - "query_comment": "100% sampling" + "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS" } }, "MIDTracks": { @@ -58,11 +49,9 @@ "moduleName": "QcMID", "detectorName": "MID", "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", "dataSource": { "type": "direct", - "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS", - "query_comment": "100% sampling" + "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS" } } }, @@ -124,4 +113,4 @@ } }, "dataSamplingPolicies": [] -} \ No newline at end of file +} From 714fa53d4ea72ac1a33737adad40bf3ef39bda5e Mon Sep 17 00:00:00 2001 From: ValerieRamillien <95756671+ValerieRamillien@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:35:07 +0100 Subject: [PATCH 0345/1239] Update mid.json correction --- DATA/production/qc-async/mid.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DATA/production/qc-async/mid.json b/DATA/production/qc-async/mid.json index 7502d9f30..4f49f2ce9 100644 --- a/DATA/production/qc-async/mid.json +++ b/DATA/production/qc-async/mid.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080" + "host": "ccdb-test.cern.ch:8080" }, "Activity": { }, @@ -27,7 +27,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "digits:MID/DATA;digits_rof:MID/DATAROF" + "query": "digits:MID/DATA;digits_rof:MID/DATAROF" } }, "MIDClusters": { @@ -39,7 +39,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS" + "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS" } }, "MIDTracks": { @@ -51,7 +51,7 @@ "cycleDurationSeconds": "60", "dataSource": { "type": "direct", - "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS" + "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS" } } }, From af9d2a8c221a004c51b4222e5a0e51bf8dd3215e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Jacazio?= Date: Tue, 6 Feb 2024 09:54:04 +0100 Subject: [PATCH 0346/1239] TOF: add good match in anQC (#1425) * TOF: add good match in anQC * Update analysis-testing-data.json --- .../default/pbpb/analysis-testing-data.json | 235 +++++++++++++++--- .../default/pp/analysis-testing-data.json | 9 +- .../json/default/pp/analysis-testing-mc.json | 6 +- 3 files changed, 217 insertions(+), 33 deletions(-) diff --git a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json index 3b8c49112..648b471f3 100644 --- a/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pbpb/analysis-testing-data.json @@ -54,8 +54,10 @@ "processMFT": "true" }, "bc-selection-task": { + "ITSROFrameBorderMargin": "30", "processRun2": "false", - "processRun3": "true" + "processRun3": "true", + "triggerBcShift": "999" }, "cell-monitor": { "maxCellTimeMain": "100", @@ -128,12 +130,14 @@ "processRun3": "true" }, "event-selection-task": { + "customDeltaBC": "-1", "isMC": "false", "muonSelection": "0", "processRun2": "false", "processRun3": "true", - "syst": "pp" + "syst": "PbPb" }, + "ft0-corrected-table": "", "lambdakzero-builder": { "ccdb-url": "http://alice-ccdb.cern.ch", "createV0CovMats": "-1", @@ -165,6 +169,59 @@ "v0radius": "1" }, "multiplicity-table": { + "ccdbpath": "Centrality/Calibration", + "ccdburl": "http://alice-ccdb.cern.ch", + "doVertexZeq": "1", + "enabledTables": { + "labels_cols": [ + "Enable" + ], + "labels_rows": [ + "FV0Mults", + "FT0Mults", + "FDDMults", + "ZDCMults", + "TrackletMults", + "TPCMults", + "PVMults", + "MultsExtra", + "MultZeqs", + "MultsExtraMC" + ], + "values": [ + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ] + ] + }, + "fractionOfEvents": "2", "processRun2": "false", "processRun3": "true" }, @@ -293,10 +350,125 @@ "rapidity": "0.5", "v0cospa": "0.995" }, - "tof-pid-qa": { + "timestamp-task": {}, + "tof-event-time": { + "ccdb-timestamp": "-1", + "ccdb-url": "", + "enableTimeDependentResponse": "false", + "fatalOnPassNotAvailable": "true", + "inheritFromBaseTask": "true", + "loadResponseFromCCDB": "false", + "maxEvTimeTOF": "100000", + "maxMomentum": "2", + "maxNtracksInSet": "10", + "minMomentum": "0.5", + "paramFileName": "", + "parametrizationPath": "TOF/Calib/Params", + "passName": "", + "processFT0": "true", + "processNoFT0": "false", + "processOnlyFT0": "false", + "processRun2": "false", + "sel8TOFEvTime": "false" + }, + "tof-pid": { + "ccdb-timestamp": "-1", + "ccdb-url": "", + "enableParticle": { + "labels_cols": [ + "Enable" + ], + "labels_rows": [ + "El", + "Mu", + "Pi", + "Ka", + "Pr", + "De", + "Tr", + "He", + "Al" + ], + "values": [ + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ], + [ + "-1" + ] + ] + }, + "enableTimeDependentResponse": "false", + "fatalOnPassNotAvailable": "true", + "inheritFromBaseTask": "true", + "loadResponseFromCCDB": "false", + "paramFileName": "", + "parametrizationPath": "", + "passName": "", + "processWSlice": "true", + "processWoSlice": "false", + "timeShiftCCDBPath": "" + }, + "tof-pid-beta": {}, + "tof-pid-beta-qa": { "applyEvSel": "2", - "applyRapidityCut": "1", + "lastTrdLayerForTrdMatch": "5", + "logAxis": "0", + "maxP": "5", + "minP": "0.100000001", + "nBinsP": "400", "requireGoodMatchTracks": "true", + "splitSignalPerCharge": "true", + "splitSignalPerEvTime": "true", + "splitTrdTracks": "false", + "tofBetaBins": { + "values": [ + "4000", + "0", + "2" + ] + }, + "tofMassBins": { + "values": [ + "1000", + "0", + "3" + ] + }, + "trackLengthBins": { + "values": [ + "100", + "0", + "1000" + ] + }, + "trackSelection": "1" + }, + "tof-pid-full": {}, + "tof-pid-qa": { + "applyEvSel": "2", + "applyRapidityCut": "true", "deltaBins": { "values": [ 100, @@ -304,8 +476,8 @@ 1000 ] }, - "enableEvTimeSplitting": "0", - "enableVsMomentumHistograms": "0", + "enableEvTimeSplitting": "false", + "enableVsMomentumHistograms": "false", "etaBins": { "values": [ 50, @@ -338,32 +510,33 @@ 6.283185307179586 ] }, - "processAlpha": "0", - "processDeuteron": "0", - "processElectron": "0", - "processFullAlpha": "0", - "processFullDeuteron": "1", - "processFullElectron": "0", - "processFullHelium3": "1", - "processFullKaon": "1", - "processFullMuon": "0", - "processFullPion": "1", - "processFullProton": "1", - "processFullTriton": "1", - "processHelium3": "0", - "processKaon": "0", - "processMuon": "0", - "processPion": "0", - "processProton": "0", - "processTriton": "0", - "produceDeltaTEtaPhiMap": "0", + "processAlpha": "false", + "processDeuteron": "false", + "processElectron": "false", + "processFullAlpha": "false", + "processFullDeuteron": "true", + "processFullElectron": "false", + "processFullHelium3": "true", + "processFullKaon": "true", + "processFullMuon": "false", + "processFullPion": "true", + "processFullProton": "true", + "processFullTriton": "false", + "processHelium3": "false", + "processKaon": "false", + "processMuon": "false", + "processPion": "false", + "processProton": "false", + "processTriton": "false", + "produceDeltaTEtaPhiMap": "false", "ptDeltaTEtaPhiMap": "3", - "splitSignalPerCharge": "1", + "requireGoodMatchTracks": "true", + "splitSignalPerCharge": "true", "trackLengthBins": { "values": [ - 50, - 0, - 1000 + "50", + "0", + "1000" ] }, "trackSelection": "1" @@ -500,6 +673,8 @@ "lutPath": "GLO/Param/MatLUT", "mVtxPath": "GLO/Calib/MeanVertex", "processCovariance": "true", + "processStandardWithPID": "false", + "processCovarianceWithPID": "false", "processStandard": "false" }, "track-selection": { @@ -517,4 +692,4 @@ "ft0-qa": { "isLowFlux": "false" } -} +} \ No newline at end of file diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json index 817d56da9..b41440ae8 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-data.json @@ -54,8 +54,10 @@ "processMFT": "true" }, "bc-selection-task": { + "ITSROFrameBorderMargin": "30", "processRun2": "false", - "processRun3": "true" + "processRun3": "true", + "triggerBcShift": "999" }, "cell-monitor": { "maxCellTimeMain": "100", @@ -128,6 +130,7 @@ "processRun3": "true" }, "event-selection-task": { + "customDeltaBC": "-1", "isMC": "false", "muonSelection": "0", "processRun2": "false", @@ -489,6 +492,8 @@ "lutPath": "GLO/Param/MatLUT", "mVtxPath": "GLO/Calib/MeanVertex", "processCovariance": "true", + "processStandardWithPID": "false", + "processCovarianceWithPID": "false", "processStandard": "false" }, "track-selection": { @@ -506,4 +511,4 @@ "ft0-qa": { "isLowFlux": "true" } -} +} \ No newline at end of file diff --git a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json index 4ab7676e0..8c9bf7005 100644 --- a/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json +++ b/MC/config/analysis_testing/json/default/pp/analysis-testing-mc.json @@ -1366,15 +1366,19 @@ "lutPath": "GLO/Param/MatLUT", "mVtxPath": "GLO/Calib/MeanVertex", "processCovariance": "true", - "processStandard": "false" + "processCovarianceWithPID": "false", + "processStandard": "false", + "processStandardWithPID": "false" }, "track-selection": { "compatibilityIU": "false", + "dcaSetup": "0", "etaMax": "0.8", "etaMin": "-0.8", "isRun3": "true", "itsMatching": "1", "produceFBextendedTable": "-1", + "produceTable": "-1", "ptMax": "1e+10", "ptMin": "0.1" }, From a64891996a8d33ddfd800eeef758fb0c16545a29 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 6 Feb 2024 15:47:47 +0100 Subject: [PATCH 0347/1239] Adjusting the value --- DATA/production/configurations/asyncReco/setenv_extra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index bc0797b71..6c8397ed3 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -608,7 +608,7 @@ if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then fi if [[ $ON_SKIMMED_DATA == 1 ]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then if [[ $ON_SKIMMED_DATA == 1 ]] ; then - SAMPLINGFACTORTIMESERIES=0.1f + SAMPLINGFACTORTIMESERIES=0.1 fi if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority export SAMPLINGFACTORTIMESERIES=${ALIEN_JDL_SAMPLINGFACTORTIMESERIES} From 861db706d79da6fc40db3826ff8e3a807be3a42f Mon Sep 17 00:00:00 2001 From: Laura Serksnyte Date: Fri, 26 Jan 2024 16:18:10 +0100 Subject: [PATCH 0348/1239] Include QC on the cluster information per track --- DATA/production/qc-async/tpc.json | 19 +++++++++++++++++++ MC/config/QC/json/tpc-qc-standard-direct.json | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/DATA/production/qc-async/tpc.json b/DATA/production/qc-async/tpc.json index 6afa9aacc..350ed4d49 100644 --- a/DATA/production/qc-async/tpc.json +++ b/DATA/production/qc-async/tpc.json @@ -68,6 +68,25 @@ "cutMaxpTPCMIPs": "0.55" } }, + "TPCTrackClusters": { + "active": "true", + "taskName": "TrackClusters", + "className": "o2::quality_control_modules::tpc::TrackClusters", + "moduleName": "QcTPC", + "detectorName": "TPC", + "cycleDurationSeconds": "300", + "dataSource": { + "type": "direct", + "query": "inputTracks:TPC/TRACKS/0;inputClusters:TPC/CLUSTERNATIVE;inputClusRefs:TPC/CLUSREFS/0" + }, + "taskParameters": { + "cutAbsEta": "1.", + "cutMinNCluster": "60", + "cutMindEdxTot": "20.", + "seed": "0", + "samplingFraction": "0.1" + } + }, "TPCTracks": { "active": "true", "taskName": "Tracks", diff --git a/MC/config/QC/json/tpc-qc-standard-direct.json b/MC/config/QC/json/tpc-qc-standard-direct.json index a1ab6692a..533bc4ea5 100644 --- a/MC/config/QC/json/tpc-qc-standard-direct.json +++ b/MC/config/QC/json/tpc-qc-standard-direct.json @@ -58,6 +58,24 @@ "TimeBinXMax": "100000" } }, + "TrackClusters": { + "active": "true", + "className": "o2::quality_control_modules::tpc::TrackClusters", + "moduleName": "QcTPC", + "detectorName": "TPC", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "inputTracks:TPC/TRACKS/0;inputClusters:TPC/CLUSTERNATIVE;inputClusRefs:TPC/CLUSREFS/0" + }, + "taskParameters": { + "cutAbsEta": "1.", + "cutMinNCluster": "60", + "cutMindEdxTot": "20.", + "seed": "0", + "samplingFraction": "0.1" + } + }, "PID": { "active": "true", "className": "o2::quality_control_modules::tpc::PID", From 9e700b48c15e2318f0dbd3a5cf0266264c2f1446 Mon Sep 17 00:00:00 2001 From: shahoian Date: Wed, 7 Feb 2024 17:27:09 +0100 Subject: [PATCH 0349/1239] Adjust syst errors and ITS/TPC matching params, use IDCCCDB as default scaling src. 1) removed TPC cluster additive errors 2) reduces Z and tgL track syst errors 3) decreased matching chi2 cut and allowed max abs difference between TPC and ITS q/pt 4) set the matching reference X = 60 5) default TPCSCALINGSOURCE changed from CTP to IDCCCDB --- .../configurations/asyncReco/setenv_extra.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 6c8397ed3..a34d8ffce 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -293,8 +293,8 @@ elif [[ $ALIGNLEVEL == 1 ]]; then ERROB="100e-8" [[ -z $TPCITSTIMEERR ]] && TPCITSTIMEERR="0.2" [[ -z $ITS_CONFIG || "$ITS_CONFIG" != *"--tracking-mode"* ]] && export ITS_CONFIG+=" --tracking-mode async" - CUT_MATCH_CHI2=160 - export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=5;tpcitsMatch.crudeNSigma2Cut[0]=100;tpcitsMatch.crudeNSigma2Cut[1]=100;tpcitsMatch.crudeNSigma2Cut[2]=100;tpcitsMatch.crudeNSigma2Cut[3]=100;tpcitsMatch.crudeNSigma2Cut[4]=100;" + CUT_MATCH_CHI2=80 + export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.XMatchingRef=60.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=1.5;tpcitsMatch.crudeNSigma2Cut[0]=64;tpcitsMatch.crudeNSigma2Cut[1]=64;tpcitsMatch.crudeNSigma2Cut[2]=64;tpcitsMatch.crudeNSigma2Cut[3]=64;tpcitsMatch.crudeNSigma2Cut[4]=64;" #-------------------------------------- TPC corrections ----------------------------------------------- # we need to provide to TPC @@ -302,7 +302,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then # 2) what to use for corrections scaling (lumi or IDC scalers or no scaling at all) : TPC_SCALING_SOURCE # the default is to use CTP, unless specified differently in the JDL... INST_IR_FOR_TPC=${ALIEN_JDL_INSTIRFORTPC-CTP} - TPC_SCALING_SOURCE=${ALIEN_JDL_TPCSCALINGSOURCE-CTP} + TPC_SCALING_SOURCE=${ALIEN_JDL_TPCSCALINGSOURCE-IDCCCDB} # MEAN_IR_FOR_TPC allows (1) to alter the map mean IR if >0 or (2) disable all corrections if <0 MEAN_IR_FOR_TPC=${ALIEN_JDL_MEANIRFORTPC-} @@ -408,15 +408,15 @@ elif [[ $ALIGNLEVEL == 1 ]]; then echo "We are in period $PERIOD, we need to keep the correction for the TPC cluster time, since no new vdrift was extracted" fi - TRACKTUNETPCINNER="trackTuneParams.tpcCovInnerType=1;trackTuneParams.tpcCovInner[0]=0.1;trackTuneParams.tpcCovInner[1]=1.;trackTuneParams.tpcCovInner[2]=6.3e-4;trackTuneParams.tpcCovInner[3]=6.3e-3;trackTuneParams.tpcCovInner[4]=2.6e-3;" - TRACKTUNETPCOUTER="trackTuneParams.tpcCovOuterType=1;trackTuneParams.tpcCovOuter[0]=0.1;trackTuneParams.tpcCovOuter[1]=1.;trackTuneParams.tpcCovOuter[2]=6.3e-4;trackTuneParams.tpcCovOuter[3]=6.3e-3;trackTuneParams.tpcCovOuter[4]=2.6e-3;" + TRACKTUNETPCINNER="trackTuneParams.tpcCovInnerType=1;trackTuneParams.tpcCovInner[0]=0.1;trackTuneParams.tpcCovInner[1]=0.2;trackTuneParams.tpcCovInner[2]=6.e-4;trackTuneParams.tpcCovInner[3]=6.e-4;trackTuneParams.tpcCovInner[4]=2.6e-3;" + TRACKTUNETPCOUTER="trackTuneParams.tpcCovOuterType=1;trackTuneParams.tpcCovOuter[0]=0.1;trackTuneParams.tpcCovOuter[1]=0.2;trackTuneParams.tpcCovOuter[2]=6.e-4;trackTuneParams.tpcCovOuter[3]=6.e-4;trackTuneParams.tpcCovOuter[4]=2.6e-3;" fi # adding additional cluster errors # the values below should be squared, but the validation of those values (0.01 and 0.0225) is ongoing -TPCEXTRAERR=";GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;" -TRACKTUNETPC="$TPCEXTRAERR" +#TPCEXTRAERR=";GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;" +TRACKTUNETPC=${TPCEXTRAERR-} # combining parameters [[ ! -z ${TRACKTUNETPCINNER:-} || ! -z ${TRACKTUNETPCOUTER:-} ]] && TRACKTUNETPC="$TRACKTUNETPC;trackTuneParams.sourceLevelTPC=true;$TRACKTUNETPCINNER;$TRACKTUNETPCOUTER" From 7e20a7fa5011fd5878b4eea4b88bfa5e0ef51c94 Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 5 Feb 2024 16:46:31 +0100 Subject: [PATCH 0350/1239] Utility to produce and analyse file-task graph relationships A small set of tools allowing to produce a report of which O2DPG MC task reads and produces which files --- UTILS/FileIOGraph/README.md | 38 +++++ UTILS/FileIOGraph/analyse_FileIO.py | 193 +++++++++++++++++++++++ UTILS/FileIOGraph/monitor_fileaccess.cpp | 188 ++++++++++++++++++++++ 3 files changed, 419 insertions(+) create mode 100644 UTILS/FileIOGraph/README.md create mode 100755 UTILS/FileIOGraph/analyse_FileIO.py create mode 100644 UTILS/FileIOGraph/monitor_fileaccess.cpp diff --git a/UTILS/FileIOGraph/README.md b/UTILS/FileIOGraph/README.md new file mode 100644 index 000000000..48443b310 --- /dev/null +++ b/UTILS/FileIOGraph/README.md @@ -0,0 +1,38 @@ +This is a small custom tool to monitor file access +and to produce graphs of file production and file consumption +by O2DPG Monte Carlo tasks. Such information can be useful for + +(a) verification of data paths +(b) early removal of files as soon as they are not needed anymore + + +In more detail, core elements of this directory are + +* monitor_fileaccess: + +A tool, useable by root, providing reports about +read and write events to files and which process is involved. +The tool is based on the efficient fanotify kernel system and reporting +can be restricted to certain shells (by giving a mother PID). + +The tool is standalone and can be compiled, if needed, by running + +`g++ monitor_fileaccess.cpp -O2 -o monitor_fileaccess.exe` + +The tool can be run simply by + +``` +sudo MAXMOTHERPID=689584 ./monitor.exe | tee /tmp/fileaccess +``` + +to monitor file events happening by child processes of shell 689584. + + +* analyse_FileIO.py: + + + + + + + diff --git a/UTILS/FileIOGraph/analyse_FileIO.py b/UTILS/FileIOGraph/analyse_FileIO.py new file mode 100755 index 000000000..5ad4523f9 --- /dev/null +++ b/UTILS/FileIOGraph/analyse_FileIO.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 + +# This is a python script which analyses +# a report from a "fanotify" file access report +# convoluted with task information from an O2DPG MC workflow. +# The tool produces: +# - a json report +# - optionally a graphviz visualization of file and task dependencies + +import argparse +import re +import json + +try: + from graphviz import Digraph + havegraphviz=True +except ImportError: + havegraphviz=False + +parser = argparse.ArgumentParser(description='Produce O2DPG MC file dependency reports') + +# the run-number of data taking or default if unanchored +parser.add_argument('--actionFile', type=str, help="O2DPG pipeline runner action file") +parser.add_argument('--monitorFile', type=str, help="monitoring file provided by fanotify tool. See O2DPG/UTILS/FileIOGraph.") +parser.add_argument('--basedir', type=str, help="O2DPG workflow dir") +parser.add_argument('--file-filters', nargs='+', default=[r'.*'], help="Filters (regular expressions) to select files (default all = '.*')") +parser.add_argument('--graphviz', type=str, help="Produce a graphviz plot") +parser.add_argument('-o','--output', type=str, help="Output JSON report") + +args = parser.parse_args() + +# what do we need to do +# (a) - parse action File for mapping of O2DPG task name to PID +# ---> fills pid_to_task + task_to_pid + +# Define the pattern using regular expressions +pid_to_O2DPGtask = {} +O2DPGtask_to_pid = {} + +pattern = re.compile(r'.*INFO Task (\d+).*:(\w+) finished with status 0') +# Open the action file and process each line +with open(args.actionFile, 'r') as file: + for line in file: + # Try to match the pattern in each line + match = pattern.match(line) + + # If a match is found, extract the information + if match: + task_number = match.group(1) + task_name = match.group(2) + + pid_to_O2DPGtask[task_number] = task_name + O2DPGtask_to_pid[task_name] = task_number + + +# (b) - parse monitor file for mapping from files to processes and operation +# ---> fills the following structures: +task_reads = { tname : set() for tname in O2DPGtask_to_pid } +task_writes = { tname : set() for tname in O2DPGtask_to_pid } +file_written_task = {} +file_consumed_task = {} + +pattern = re.compile(args.basedir + r'([^,]+),((?:read|write)),(.*)') +# neglecting some framework file names +file_exclude_filter = re.compile(r'(.*)\.log(.*)|(ccdb/log)|(.*)dpl-config\.json') + +# construct user-filter regular expressions +file_filter_re = [ re.compile(l) for l in args.file_filters ] + +with open(args.monitorFile, 'r') as file: + for line in file: + # Try to match the pattern in each line + match = pattern.match(line) + if match: + file_name = match.group(1) + mode = match.group(2) + pids = match.group(3).split(";") + + # implement file name filter + if file_exclude_filter.match(file_name): + continue + + # look if file matches one of the user provided filters + file_matches = False + for r in file_filter_re: + if r.match(file_name): + file_matches = True + break + + if not file_matches: + continue + + if file_consumed_task.get(file_name) == None: + file_consumed_task[file_name] = set() + if file_written_task.get(file_name) == None: + file_written_task[file_name] = set() + + for p in pids: + if p in pid_to_O2DPGtask: + task = pid_to_O2DPGtask.get(p) + if mode == 'read': + task_reads.get(task).add(file_name) + file_consumed_task[file_name].add(task) + + if mode == 'write': + task_writes.get(task).add(file_name) + file_written_task[file_name].add(task) + + +# draws the graph of files and tasks +def draw_graph(graphviz_filename): + if not havegraphviz: + print('graphviz not installed, cannot draw workflow') + return + + dot = Digraph(comment='O2DPG file - task network') + + ccdbfilter = re.compile('ccdb(.*)/snapshot.root') + + nametoindex={} + index=0 + + allfiles = set(file_written_task.keys()) | set(file_consumed_task.keys()) + normalfiles = [ s for s in allfiles if not ccdbfilter.match(s) ] + ccdbfiles = [ (s, ccdbfilter.match(s).group(1)) for s in allfiles if ccdbfilter.match(s) ] + + with dot.subgraph(name='CCDB') as ccdbpartition: + ccdbpartition.attr(color = 'blue') + for f in ccdbfiles: + nametoindex[f[0]] = index + ccdbpartition.node(str(index), f[1], color = 'blue') + index = index + 1 + + with dot.subgraph(name='normal') as normalpartition: + normalpartition.attr(color = 'black') + for f in normalfiles: + nametoindex[f] = index + normalpartition.node(str(index), f, color = 'red') + index = index + 1 + for t in O2DPGtask_to_pid: + nametoindex[t] = index + normalpartition.node(str(index), t, shape = 'box', color = 'green', style = 'filled' ) + index = index + 1 + + # edges (arrows between files and tasks) + for node in file_consumed_task: + # node is a file (source) + sourceindex = nametoindex[node] + for task in file_consumed_task[node]: + toindex = nametoindex[task] + dot.edge(str(sourceindex), str(toindex)) + + # edges (arrows between files and tasks) + for node in file_written_task: + # node is a file (target) + toindex = nametoindex[node] + for task in file_written_task[node]: + sourceindex = nametoindex[task] + dot.edge(str(sourceindex), str(toindex)) + + dot.render(graphviz_filename, format='pdf') + dot.render(graphviz_filename, format='gv') + +def write_json_report(json_file_name): + # produce a JSON report of file dependencies + all_filenames = set(file_written_task.keys()) | set(file_consumed_task.keys()) + file_written_task_tr = [ + { + "file" : k, + "written_by" : list(file_written_task.get(k, [])), + "read_by" : list(file_consumed_task.get(k, [])) + } + for k in all_filenames + ] + + tasks_output = [ + { + "task" : t, + "writes" : list(task_writes.get(t,[])), + "reads" : list(task_reads.get(t,[])) + } + for t in O2DPGtask_to_pid + ] + + # Write the dictionary to a JSON file + with open(json_file_name, 'w') as json_file: + json.dump({ "file_report" : file_written_task_tr, "task_report" : tasks_output }, json_file, indent=2) + +if args.graphviz: + draw_graph(args.graphviz) + +if args.output: + write_json_report(args.output) \ No newline at end of file diff --git a/UTILS/FileIOGraph/monitor_fileaccess.cpp b/UTILS/FileIOGraph/monitor_fileaccess.cpp new file mode 100644 index 000000000..50567e593 --- /dev/null +++ b/UTILS/FileIOGraph/monitor_fileaccess.cpp @@ -0,0 +1,188 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#define CHK(expr, errcode) \ + if ((expr) == errcode) \ + perror(#expr), exit(EXIT_FAILURE) + +#include +#include +#include +#include +#include +#include + +#define MAXBUF (BUFSIZ * 2) + +int getppid(int pid) +{ + int ppid; + char buf[MAXBUF]; + char procname[32]; // Holds /proc/4294967296/status\0 + FILE* fp; + + snprintf(procname, sizeof(procname), "/proc/%u/status", pid); + fp = fopen(procname, "r"); + if (fp != NULL) { + size_t ret = fread(buf, sizeof(char), MAXBUF - 1, fp); + if (!ret) { + return 0; + } else { + buf[ret++] = '\0'; // Terminate it. + } + fclose(fp); + } + char* ppid_loc = strstr(buf, "\nPPid:"); + if (ppid_loc) { + int ret = sscanf(ppid_loc, "\nPPid:%d", &ppid); + if (!ret || ret == EOF) { + return 0; + } + return ppid; + } else { + return 0; + } +} + +std::string getcmd(pid_t pid) +{ + char path[1024]; + snprintf(path, sizeof(path), "/proc/%d/cmdline", pid); + if (pid == 0 || pid == 1) { + return std::string(""); + } + + FILE* file = fopen(path, "r"); + if (file) { + char buffer[1024]; // max 1024 chars + size_t bytesRead = fread(buffer, 1, sizeof(buffer), file); + fclose(file); + for (int byte = 0; byte < bytesRead; ++byte) { + if (buffer[byte] == '\0') { + buffer[byte] == '@'; + } + } + return std::string(buffer); + } + return std::string(""); +} + +std::unordered_map good_pid; + +bool is_good_pid(int pid, int maxparent) +{ + auto iter = good_pid.find(pid); + if (iter != good_pid.end()) { + // the result is known + return iter->second; + } + // the result is not known ---> determine it + + // this means determining the whole chain of parent ids + if (pid == maxparent) { + good_pid[pid] = true; + } else if (pid == 0) { + good_pid[pid] = false; + } else { + good_pid[pid] = is_good_pid(getppid(pid), maxparent); + } + return good_pid[pid]; +} + +int main(int argc, char** argv) +{ + int fan; + char buf[4096]; + char fdpath[32]; + char path[PATH_MAX + 1]; + ssize_t buflen, linklen; + struct fanotify_event_metadata* metadata; + + CHK(fan = fanotify_init(FAN_CLASS_NOTIF, O_RDONLY), -1); + CHK(fanotify_mark(fan, FAN_MARK_ADD | FAN_MARK_MOUNT, + FAN_CLOSE_WRITE | FAN_CLOSE_NOWRITE | FAN_EVENT_ON_CHILD, AT_FDCWD, "/"), + -1); + + std::unordered_map pid_to_parents; // mapping of a process id to the whole string of parent pids, separated by ';' + std::unordered_map pid_to_command; // mapping of a process id to a command + + auto MAX_MOTHER_PID_ENV = getenv("MAXMOTHERPID"); + int max_mother_pid = 1; // everything + if (MAX_MOTHER_PID_ENV != nullptr) { + std::cerr << "found env variablen"; + max_mother_pid = std::atoi(MAX_MOTHER_PID_ENV); + std::cerr << "Setting topmost mother process to " << max_mother_pid << "\n"; + } else { + std::cerr << "No environment given\n"; + } + + auto thispid = getpid(); + std::string* parentspid = nullptr; + + for (;;) { + CHK(buflen = read(fan, buf, sizeof(buf)), -1); + metadata = (struct fanotify_event_metadata*)&buf; + while (FAN_EVENT_OK(metadata, buflen)) { + if (metadata->mask & FAN_Q_OVERFLOW) { + printf("Queue overflow!\n"); + continue; + } + sprintf(fdpath, "/proc/self/fd/%d", metadata->fd); + CHK(linklen = readlink(fdpath, path, sizeof(path) - 1), -1); + path[linklen] = '\0'; + auto pid = metadata->pid; + + bool record = true; + + // no need to monitor ourselfs + record = record && pid != thispid; + + // check if we have the right events before continuing + record = record && (((metadata->mask & FAN_CLOSE_WRITE) || (metadata->mask & FAN_CLOSE_NOWRITE))); + + // check if we have the right pid before continuing + record = record && is_good_pid(pid, max_mother_pid); + + if (record) { + auto iter = pid_to_parents.find((int)pid); + if (iter != pid_to_parents.end()) { + parentspid = &iter->second; + } else { + std::stringstream str; + // get chain of parent pids + auto current = (int)pid; + str << current; + while (current != max_mother_pid && current != 0) { + // record command line of current if not already cached + if (pid_to_command.find((int)current) == pid_to_command.end()) { + std::string cmd{getcmd(current)}; + pid_to_command[current] = cmd; + printf("pid-to-command:%i:%s\n", current, cmd.c_str()); + } + + auto next = getppid(current); + current = next; + str << ";" << current; + } + pid_to_parents[(int)pid] = str.str(); + parentspid = &pid_to_parents[(int)pid]; + } + + if (metadata->mask & FAN_CLOSE_WRITE) { + printf("%s,write,%s\n", path, parentspid->c_str()); + } + if (metadata->mask & FAN_CLOSE_NOWRITE) { + printf("%s,read,%s\n", path, parentspid->c_str()); + } + } + + close(metadata->fd); + metadata = FAN_EVENT_NEXT(metadata, buflen); + } + } +} From ac9549fd177e22821f0aca18ba6d5d7870e87199 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Thu, 8 Feb 2024 16:35:12 +0100 Subject: [PATCH 0351/1239] Make reading patterns from file more resilient (#1450) Co-authored-by: Benedikt Volkel --- RelVal/utils/o2dpg_release_validation_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/RelVal/utils/o2dpg_release_validation_utils.py b/RelVal/utils/o2dpg_release_validation_utils.py index 633e424c6..ca1d786f5 100755 --- a/RelVal/utils/o2dpg_release_validation_utils.py +++ b/RelVal/utils/o2dpg_release_validation_utils.py @@ -280,8 +280,15 @@ def set_object_name_patterns(self, include_patterns, exclude_patterns): def load_this_patterns(patterns): if not patterns or not patterns[0].startswith("@"): return patterns + + patterns_from_file = [] with open(patterns[0][1:], "r") as f: - return f.read().splitlines() + for line in f: + line = line.strip() + if not line: + continue + patterns_from_file.append(line) + return patterns_from_file self.include_patterns = load_this_patterns(include_patterns) self.exclude_patterns = load_this_patterns(exclude_patterns) From a742df325715628593f7648ba7dfa14a4b5daa38 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Fri, 9 Feb 2024 16:20:19 +0100 Subject: [PATCH 0352/1239] Update TRD QC json files (#1448) --- DATA/production/qc-async/trd.json | 10 ++-------- DATA/production/qc-sync/trd.json | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index d32e1fbfd..3d9633ea1 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -4,22 +4,16 @@ "database": { "implementation": "CCDB", "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" }, "Activity": { "type": "2", "number": "42" }, "monitoring": { - "url": "influxdb-unix:///tmp/telegraf.sock" - }, - "consul": { - "url": "alio2-cr1-hv-aliecs:8500" + "url": "infologger:///debug?qc" }, "conditionDB": { - "url": "alio2-cr1-hv-qcdb1.cern.ch:8083" + "url": "ccdb-test.cern.ch:8080" } }, "tasks": { diff --git a/DATA/production/qc-sync/trd.json b/DATA/production/qc-sync/trd.json index a47867470..4895e8c85 100644 --- a/DATA/production/qc-sync/trd.json +++ b/DATA/production/qc-sync/trd.json @@ -22,7 +22,6 @@ "url": "o2-ccdb.internal" }, "infologger": { - "": "Message at this level or above are discarded (default: 21 - Trace)", "filterDiscardDebug": "false", "filterDiscardLevel": "11" } @@ -33,11 +32,15 @@ "className": "o2::quality_control_modules::trd::RawData", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", + "cycleDurations": [ + {"cycleDurationSeconds": 30, "validitySeconds": 180}, + {"cycleDurationSeconds": 180, "validitySeconds": 1} + ], "dataSource": { "type": "direct", "query": "rawstats:TRD/RAWSTATS" }, + "disableLastCycle": "true", "location": "local", "localMachines": [ "epn", @@ -54,11 +57,12 @@ "className": "o2::quality_control_modules::trd::DigitsTask", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "180", "dataSource": { "type": "dataSamplingPolicy", "name": "trdall" }, + "disableLastCycle": "true", "location": "local", "localMachines": [ "epn", @@ -79,11 +83,12 @@ "className": "o2::quality_control_modules::trd::TrackletsTask", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "120", "dataSource": { "type": "dataSamplingPolicy", "name": "trdall" }, + "disableLastCycle": "true", "location": "local", "localMachines": [ "epn", @@ -105,11 +110,12 @@ "className": "o2::quality_control_modules::trd::PulseHeightTrackMatch", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "180", "dataSource": { "type": "direct", "query": "phValues:TRD/PULSEHEIGHT" }, + "disableLastCycle": "true", "location": "local", "localMachines": [ "epn", @@ -129,11 +135,12 @@ "className": "o2::quality_control_modules::trd::TrackingTask", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", + "cycleDurationSeconds": "180", "dataSource": { "type": "direct", "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC" }, + "disableLastCycle": "true", "location": "local", "localMachines": [ "epn", From d17fdfb923737857897bfca4d069183ffa969fc9 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 9 Feb 2024 17:14:37 +0100 Subject: [PATCH 0353/1239] Fixes to make the split wf work again (#1451) --- DATA/common/gen_topo_helper_functions.sh | 12 ++++- DATA/common/setenv.sh | 12 +++++ DATA/common/setenv_calib.sh | 7 +++ .../configurations/asyncReco/async_pass.sh | 15 ++++-- .../configurations/asyncReco/setenv_extra.sh | 51 +++++++++---------- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index 6a3e84149..00fc1f118 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -9,9 +9,19 @@ has_detector() [[ $WORKFLOW_DETECTORS =~ (^|,)"$1"(,|$) ]] } +has_detector_from_global_reader_clusters() +{ + [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS =~ (^|,)"$1"(,|$) ]] +} + +has_detector_from_global_reader_tracks() +{ + [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS =~ (^|,)"$1"(,|$) ]] +} + has_detector_from_global_reader() { - [[ $WORKFLOW_DETECTORS_USE_GLOBAL_READER =~ (^|,)"$1"(,|$) ]] + has_detector_from_global_reader_tracks $1 || has_detector_from_global_reader_clusters $1 } has_detector_calib() diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 6c31539c9..f7342e553 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -49,6 +49,8 @@ if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == if [[ -z "${WORKFLOW_DETECTORS_CTF+x}" ]] || [[ "0$WORKFLOW_DETECTORS_CTF" == "0ALL" ]]; then export WORKFLOW_DETECTORS_CTF=$WORKFLOW_DETECTORS; fi if [[ "0${WORKFLOW_DETECTORS_FLP_PROCESSING:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING=$WORKFLOW_DETECTORS; fi if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER=${WORKFLOW_DETECTORS_USE_GLOBAL_READER:-}; fi +if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS:-}; fi +if [[ "0${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}" == "0ALL" ]]; then export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$WORKFLOW_DETECTORS; else export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=${WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS:-}; fi if [[ -z "${WORKFLOW_PARAMETERS:-}" ]]; then export WORKFLOW_PARAMETERS=; fi if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_QC:-} ]]; then @@ -61,6 +63,16 @@ if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_CALIB:-} ]]; then export WORKFLOW_DETECTORS_CALIB=$(echo $WORKFLOW_DETECTORS_CALIB | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") done fi +if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS:-} ]]; then + for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS//,/ }; do + export WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") + done +fi +if [[ ! -z ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS:-} ]]; then + for i in ${WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_CLUSTERS//,/ }; do + export WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=$(echo $WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") + done +fi if [[ -z "${TFLOOP:-}" ]]; then export TFLOOP=0; fi # loop over timeframes if [[ -z "${NTIMEFRAMES:-}" ]]; then export NTIMEFRAMES=-1; fi # max number of time frames to process, <=0 : unlimited diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 87fe461ea..a1d2e7692 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -30,6 +30,8 @@ if has_detector_calib FDD && has_processing_step FDD_RECO; then CAN_DO_CALIB_FDD if has_detector_calib ZDC && has_processing_step ZDC_RECO; then CAN_DO_CALIB_ZDC_TDC=1; else CAN_DO_CALIB_ZDC_TDC=0; fi # for async recalibration if has_detector_calib EMC && has_detector_reco EMC && [[ $SYNCMODE != 1 ]]; then CAN_DO_CALIB_EMC_ASYNC_RECALIB=1; else CAN_DO_CALIB_EMC_ASYNC_RECALIB=0; fi +if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC; then CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=1; else CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS=0; fi +if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0; then CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=1; else CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES=0; fi # additional individual settings for calibration workflows if has_detector CTP; then export CALIB_TPC_SCDCALIB_CTP_INPUT="--enable-ctp"; else export CALIB_TPC_SCDCALIB_CTP_INPUT=""; fi @@ -174,6 +176,11 @@ fi ( [[ -z ${CALIB_ZDC_TDC:-} ]] || [[ $CAN_DO_CALIB_ZDC_TDC == 0 ]] ) && CALIB_ZDC_TDC=0 # for async: ( [[ -z ${CALIB_EMC_ASYNC_RECALIB:-} ]] || [[ $CAN_DO_CALIB_EMC_ASYNC_RECALIB == 0 ]] ) && CALIB_EMC_ASYNC_RECALIB=0 +( [[ -z ${CALIB_ASYNC_EXTRACTTPCCURRENTS:-} ]] || [[ $CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS == 0 ]] ) && CALIB_ASYNC_EXTRACTTPCCURRENTS=0 +( [[ -z ${CALIB_ASYNC_DISABLE3DCURRENTS:-} ]] || [[ $CAN_DO_CALIB_ASYNC_DISABLE3DCURRENTS == 0 ]] ) && CALIB_ASYNC_DISABLE3DCURRENTS=0 +: ${ON_SKIMMED_DATA:=0} +( [[ -z ${CALIB_ASYNC_EXTRACTTIMESERIES:-} ]] || [[ $CAN_DO_CALIB_ASYNC_EXTRACTTIMESERIES == 0 ]] ) && CALIB_ASYNC_EXTRACTTIMESERIES=0 + if [[ "0${GEN_TOPO_VERBOSE:-}" == "01" ]]; then echo "CALIB_PRIMVTX_MEANVTX = $CALIB_PRIMVTX_MEANVTX" 1>&2 diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index c65ac7b49..05ed0dea2 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -149,6 +149,14 @@ if [[ -n "$ALIEN_JDL_DOMEANVTXCALIB" ]]; then export ADD_CALIB=1 fi +if [[ "$ALIEN_JDL_EXTRACTCURRENTS" == "1" ]]; then + export ADD_CALIB=1 +fi + +if [[ "$ALIEN_JDL_EXTRACTTIMESERIES" == "1" ]]; then + export ADD_CALIB=1 +fi + # AOD file size if [[ -n "$ALIEN_JDL_AODFILESIZE" ]]; then export AOD_FILE_SIZE="$ALIEN_JDL_AODFILESIZE" @@ -577,18 +585,19 @@ else if ([[ -z "$ALIEN_JDL_SSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq "all" ]]; then # 3. matching, QC, calib, AOD WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START + echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 3) matching, QC, calib, AOD" echo -e "\nStep 3) matching, QC, calib, AOD" >> workflowconfig.log export TIMEFRAME_RATE_LIMIT=0 echo "Removing detectors $DETECTORS_EXCLUDE" READER_DELAY=${ALIEN_JDL_READERDELAY:-30} export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " - echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader_workflow" - env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader" + env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then timeStart=`date +%s` - time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list exitcode=$? timeEnd=`date +%s` timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index a34d8ffce..1036128d6 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -553,6 +553,10 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_ZDC_TDC=0 export CALIB_FT0_TIMEOFFSET=0 export CALIB_TPC_SCDCALIB=0 + export CALIB_FT0_INTEGRATEDCURR=0 + export CALIB_FV0_INTEGRATEDCURR=0 + export CALIB_FDD_INTEGRATEDCURR=0 + export CALIB_TOF_INTEGRATEDCURR=0 if [[ $DO_TPC_RESIDUAL_EXTRACTION == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 @@ -582,38 +586,31 @@ if [[ $ADD_CALIB == "1" ]]; then export ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow="$ARGS_EXTRA_PROCESS_o2_calibration_trd_workflow --enable-root-output" export CALIB_TRD_GAIN=1 fi - if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then - export CCDB_POPULATOR_UPLOAD_PATH="file://$PWD" + # extra workflows in case we want to process the currents for FT0, FV0, TOF, TPC + if [[ -n $ALIEN_JDL_EXTRACTCURRENTS ]] ; then + export CALIB_FT0_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_FV0_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_FDD_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_TOF_INTEGRATEDCURR=$ALIEN_JDL_EXTRACTCURRENTS + export CALIB_ASYNC_EXTRACTTPCCURRENTS=$ALIEN_JDL_EXTRACTCURRENTS fi -fi - -# extra workflows in case we want to process the currents for FT0, FV0, TOF, TPC -if [[ $ALIEN_JDL_EXTRACTCURRENTS == 1 ]]; then - if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi - has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-ft0-integrate-cluster-workflow" - has_detector_reco FV0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-fv0-integrate-cluster-workflow" - has_detector_reco TOF && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tof-integrate-cluster-workflow" - if [[ $ALIEN_JDL_DISABLE3DCURRENTS != 1 ]]; then - export ARGS_EXTRA_PROCESS_o2_tpc_integrate_cluster_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_integrate_cluster_workflow--process-3D-currents --nSlicesTF 1" + if [[ -n $ALIEN_JDL_DISABLE3DCURRENTS ]]; then + export CALIB_ASYNC_DISABLE3DCURRENTS=$ALIEN_JDL_DISABLE3DCURRENTS fi - has_detector_reco TPC && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-integrate-cluster-workflow" -fi -# extra workflows in case we want to process the currents for time series -if [[ $ALIEN_JDL_EXTRACTTIMESERIES == 1 ]]; then - if [[ -z "${WORKFLOW_DETECTORS_RECO+x}" ]] || [[ "0$WORKFLOW_DETECTORS_RECO" == "0ALL" ]]; then export WORKFLOW_DETECTORS_RECO=$WORKFLOW_DETECTORS; fi - has_detector_reco TPC && has_detector_reco ITS && has_detector_reco FT0 && add_comma_separated ADD_EXTRA_WORKFLOW "o2-tpc-time-series-workflow" - if [[ ! -z "$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES" ]]; then - export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --enable-unbinned-root-output --sample-unbinned-tsallis --threads 1" - fi - if [[ $ON_SKIMMED_DATA == 1 ]] || [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]] ; then - if [[ $ON_SKIMMED_DATA == 1 ]] ; then - SAMPLINGFACTORTIMESERIES=0.1 + # extra workflows in case we want to process the currents for time series + if [[ -n $ALIEN_JDL_EXTRACTTIMESERIES ]] ; then + echo "Adding timeseries in setenv_extra.sh" + export CALIB_ASYNC_EXTRACTTIMESERIES=$ALIEN_JDL_EXTRACTTIMESERIES + if [[ -n $ALIEN_JDL_ENABLEUNBINNEDTIMESERIES ]]; then + export CALIB_ASYNC_ENABLEUNBINNEDTIMESERIES=$ALIEN_JDL_ENABLEUNBINNEDTIMESERIES fi - if [[ ! -z "$ALIEN_JDL_SAMPLINGFACTORTIMESERIES" ]]; then # this takes priority - export SAMPLINGFACTORTIMESERIES=${ALIEN_JDL_SAMPLINGFACTORTIMESERIES} + if [[ -n $ALIEN_JDL_SAMPLINGFACTORTIMESERIES ]]; then + export CALIB_ASYNC_SAMPLINGFACTORTIMESERIES=$ALIEN_JDL_SAMPLINGFACTORTIMESERIES fi - export ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow="$ARGS_EXTRA_PROCESS_o2_tpc_time_series_workflow --sampling-factor ${SAMPLINGFACTORTIMESERIES}" + fi + if [[ $ALIEN_JDL_DOUPLOADSLOCALLY == 1 ]]; then + export CCDB_POPULATOR_UPLOAD_PATH="file://$PWD" fi fi From a58581c6a8e25342fac65265cba6801ca2530b42 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Sun, 11 Feb 2024 19:23:37 +0100 Subject: [PATCH 0354/1239] Fix typo (#1453) --- DATA/production/qc-async/trd.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/qc-async/trd.json b/DATA/production/qc-async/trd.json index 3d9633ea1..967c5b85e 100644 --- a/DATA/production/qc-async/trd.json +++ b/DATA/production/qc-async/trd.json @@ -3,7 +3,7 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", + "host": "ccdb-test.cern.ch:8080" }, "Activity": { "type": "2", From e17e279d11ddaeb3cf9f5b944ebe98b39351a895 Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Mon, 12 Feb 2024 10:21:44 +0000 Subject: [PATCH 0355/1239] Check JSON syntax for pull requests (#1454) --- .github/workflows/check-json-syntax.yml | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/check-json-syntax.yml diff --git a/.github/workflows/check-json-syntax.yml b/.github/workflows/check-json-syntax.yml new file mode 100644 index 000000000..fae51ae62 --- /dev/null +++ b/.github/workflows/check-json-syntax.yml @@ -0,0 +1,37 @@ +--- +name: Validate JSON syntax + +# Run on any commit or PR that changes any JSON file. +'on': + push: + paths: + - '**.json' + pull_request: + paths: + - '**.json' + +permissions: {} + +jobs: + json-syntax: + name: validate syntax + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate syntax for JSON files + run: | + error=0 + readarray -d '' json_files < \ + <(find . \( -path ./.git -or -path ./DATA/testing/private \) -prune -false -or -type f -name '*.json' -print0) + for jsonf in "${json_files[@]}"; do + echo "::debug::Checking $jsonf..." + if ! errmsg=$(jq . "$jsonf" 2>&1 >/dev/null); then + error=1 + echo "Invalid JSON syntax found in $jsonf:" >&2 + printf '::error file=%s,title=%s::%s\n' "$jsonf" 'Invalid JSON syntax' "$errmsg" + fi + done + exit "$error" From f99e0fbaaaa15fc46c8431ea5af2366812012655 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Mon, 12 Feb 2024 14:12:11 +0100 Subject: [PATCH 0356/1239] Anchoring: Ability to get detector list from GRPECS (#1452) --- MC/bin/o2dpg_sim_workflow.py | 1 + MC/bin/o2dpg_sim_workflow_anchored.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 5f116bab5..aaf9f640a 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -59,6 +59,7 @@ parser.add_argument('-trigger',help='event selection: particle, external', default='') parser.add_argument('-ini',help='generator init parameters file (full paths required), for example: ${O2DPG_ROOT}/MC/config/PWGHF/ini/GeneratorHF.ini', default='') parser.add_argument('-confKey',help='generator or trigger configuration key values, for example: "GeneratorPythia8.config=pythia8.cfg;A.x=y"', default='') +parser.add_argument('--readoutDets',help='comma separated string of detectors readout (does not modify material budget - only hit creation)', default='all') parser.add_argument('-interactionRate',help='Interaction rate, used in digitization', default=-1) parser.add_argument('-bcPatternFile',help='Bunch crossing pattern file, used in digitization (a file name or "ccdb")', default='') diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 1876e4877..a50a0532a 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -121,7 +121,7 @@ def retrieve_CCDBObject_asJSON(ccdbreader, path, timestamp, objtype_external = N jsonTString = TBufferJSON.ConvertToJSON(obj, TClass.GetClass(objtype)) return json.loads(jsonTString.Data()) -def retrieve_sor_eor_fromGRPECS(ccdbreader, run_number, rct = None): +def retrieve_params_fromGRPECS(ccdbreader, run_number, rct = None): """ Retrieves start of run (sor), end of run (eor) and other global parameters from the GRPECS object, given a run number. We first need to find the right object @@ -191,8 +191,13 @@ def retrieve_sor_eor_fromGRPECS(ccdbreader, run_number, rct = None): print ("OrbitFirst", orbitFirst) # first orbit of this run print ("LastOrbit of run", orbitLast) + # Now fetch the detector list + print ("DetsReadout-Mask: ", grp["mDetsReadout"]['v']) + detList = o2.detectors.DetID.getNames(grp["mDetsReadout"]['v']) + print ("Detector list is ", detList) + # orbitReset.get(run_number) - return {"SOR": SOR, "EOR": EOR, "FirstOrbit" : orbitFirst, "LastOrbit" : orbitLast, "OrbitsPerTF" : int(grp["mNHBFPerTF"])} + return {"SOR": SOR, "EOR": EOR, "FirstOrbit" : orbitFirst, "LastOrbit" : orbitLast, "OrbitsPerTF" : int(grp["mNHBFPerTF"]), "detList" : detList} def retrieve_GRP(ccdbreader, timestamp): """ @@ -335,7 +340,7 @@ def main(): ccdbreader = CCDBAccessor(args.ccdb_url) # fetch the EOR/SOR rct_sor_eor = retrieve_sor_eor(ccdbreader, args.run_number) # <-- from RCT/Info - GLOparams = retrieve_sor_eor_fromGRPECS(ccdbreader, args.run_number, rct=rct_sor_eor) + GLOparams = retrieve_params_fromGRPECS(ccdbreader, args.run_number, rct=rct_sor_eor) if not GLOparams: print ("No time info found") sys.exit(1) @@ -419,7 +424,8 @@ def main(): # we finally pass forward to the unanchored MC workflow creation # TODO: this needs to be done in a pythonic way clearly - forwardargs += " -tf " + str(args.tf) + " --sor " + str(sor) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " + str(first_orbit) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) + forwardargs += " -tf " + str(args.tf) + " --sor " + str(sor) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " \ + + str(first_orbit) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) + ' --readoutDets ' + GLOparams['detList'] print ("forward args ", forwardargs) cmd = "${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py " + forwardargs print ("Creating time-anchored workflow...") From ce628a667b644f5e9cc00a0616f4677c34d88770 Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 13 Feb 2024 11:18:06 +0100 Subject: [PATCH 0357/1239] Restore cluster syst errors for PbPb only --- DATA/production/configurations/asyncReco/setenv_extra.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 1036128d6..bc65f87fb 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -384,7 +384,10 @@ elif [[ $ALIGNLEVEL == 1 ]]; then export TPC_CORR_SCALING+=" --enable-M-shape-correction " fi - if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]] ; then + if [[ $ALIEN_JDL_LPMANCHORYEAR == "2023" ]] && [[ $BEAMTYPE == "PbPb" ]] ; then + # adding additional cluster errors + # the values below should be squared, but the validation of those values (0.01 and 0.0225) is ongoing + TPCEXTRAERR=";GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;" if [[ $SCALE_WITH_ZDC == 1 ]]; then echo "For 2023 PbPb ZDC inst. lumi applying factor 2.414" export TPC_CORR_SCALING+=";TPCCorrMap.lumiInstFactor=2.414;" @@ -413,9 +416,6 @@ elif [[ $ALIGNLEVEL == 1 ]]; then fi -# adding additional cluster errors -# the values below should be squared, but the validation of those values (0.01 and 0.0225) is ongoing -#TPCEXTRAERR=";GPU_rec_tpc.clusterError2AdditionalY=0.1;GPU_rec_tpc.clusterError2AdditionalZ=0.15;" TRACKTUNETPC=${TPCEXTRAERR-} # combining parameters From d71860acc608c1ecbe07d0780f75502469c21e5f Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 13 Feb 2024 15:02:21 +0100 Subject: [PATCH 0358/1239] [RelVal] Fix index (#1456) --- RelVal/utils/o2dpg_release_validation_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RelVal/utils/o2dpg_release_validation_utils.py b/RelVal/utils/o2dpg_release_validation_utils.py index ca1d786f5..0482569aa 100755 --- a/RelVal/utils/o2dpg_release_validation_utils.py +++ b/RelVal/utils/o2dpg_release_validation_utils.py @@ -400,7 +400,7 @@ def load(self, summaries_to_test): idx, metric = self.get_metric_checking_dict(line) if idx is None: # in this case, this metric is new - idx = len(self.metrics) - 1 + idx = len(self.metrics) if not self.add_metric(metric): # only attempt to add if that metric is not yet there continue From 45852470c738a1cf04f827371349f7a9a57ee8f1 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 13 Feb 2024 16:04:17 +0100 Subject: [PATCH 0359/1239] [SimWF] Allow any detectors (#1441) * always run all digi stages independent of which detectors produce hits * run all reco and match stages on top of that * update TOF match sources * add TRD and HMP sources * introduces implicit modularisation * empty digit and reco files are always produced for missing detector hit files --> technicannly allows for any detector composition --> in particular crucial for anchored simulations --> enables the simulation of single detectors for specific expert studies when other material budget is not needed or has to be avoided even * do not hard-code ALIEN_JDL_WORKFLOWDETECTORS but actually allow to take it from the outside JDL in anchorMC.sh Co-authored-by: Benedikt Volkel --- MC/bin/o2dpg_sim_workflow.py | 213 ++++++++++------------------ MC/bin/o2dpg_workflow_utils.py | 6 +- MC/run/ANCHOR/anchorMC.sh | 5 +- UTILS/parse-async-WorkflowConfig.py | 7 + 4 files changed, 89 insertions(+), 142 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index aaf9f640a..15f54c3bd 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -34,7 +34,7 @@ sys.path.append(join(dirname(__file__), '.', 'o2dpg_workflow_utils')) -from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector +from o2dpg_workflow_utils import createTask, createGlobalInitTask, dump_workflow, adjust_RECO_environment, isActive, activate_detector, deactivate_detector from o2dpg_qc_finalization_workflow import include_all_QC_finalization from o2dpg_sim_config import create_sim_config, create_geant_config, constructConfigKeyArg @@ -194,14 +194,36 @@ def load_external_config(configfile): print(f"INFO: Written additional config key parameters to JSON {config_key_param_path}") json.dump(anchorConfig, f, indent=2) +def get_anchor_env_var(key, default): + return anchorConfig.get('env_vars',{}).get(key, default) + # with this we can tailor the workflow to the presence of # certain detectors +# these are all detectors that should be assumed active +readout_detectors = args.readoutDets +# here are all detectors that have been set in an anchored script activeDetectors = anchorConfig.get('o2-ctf-reader-workflow-options',{}).get('onlyDet','all') +if activeDetectors == 'all': + # if "all" here, there was in fact nothing in the anchored script, set to what is passed to this script (which it either also "all" or a subset) + activeDetectors = readout_detectors +elif readout_detectors != 'all' and activeDetectors != 'all': + # in this case both are comma-seperated lists. Take intersection + r = set(readout_detectors.split(',')) + a = set(activeDetectors.split(',')) + activeDetectors = ','.join(r & a) +# the last case: simply take what comes from the anchored config + # convert to set/hashmap -activeDetectors = { det:1 for det in activeDetectors.split(",") } +activeDetectors = { det:1 for det in activeDetectors.split(',') } for det in activeDetectors: activate_detector(det) +if not args.with_ZDC: + # deactivate to be able to use isActive consistently for ZDC + deactivate_detector('ZDC') + if 'ZDC' in activeDetectors: + del activeDetectors['ZDC'] + def addWhenActive(detID, needslist, appendstring): if isActive(detID): needslist.append(appendstring) @@ -291,7 +313,7 @@ def extractVertexArgs(configKeyValuesStr, finalDiamondDict): NTIMEFRAMES=int(args.tf) NWORKERS=args.j -MODULES = "--skipModules ZDC" if not args.with_ZDC else "" +MODULES = "--skipModules ZDC" if not isActive("ZDC") else "" SIMENGINE=args.e BFIELD=args.field RNDSEED=args.seed # typically the argument should be the jobid, but if we get None the current time is used for the initialisation @@ -444,7 +466,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): + ('',' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run) \ + ' --vertexMode kCCDB' - if not "all" in activeDetectors: + if not isActive('all'): BKGtask['cmd'] += ' --readoutDetectors ' + " ".join(activeDetectors) workflow['stages'].append(BKGtask) @@ -475,9 +497,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): workflow['stages'].append(BKG_HEADER_task) # a list of smaller sensors (used to construct digitization tasks in a parametrized way) -smallsensorlist = [ "ITS", "TOF", "FDD", "MCH", "MID", "MFT", "HMP", "PHS", "CPV" ] -if args.with_ZDC: - smallsensorlist += [ "ZDC" ] +smallsensorlist = [ "ITS", "TOF", "FDD", "MCH", "MID", "MFT", "HMP", "PHS", "CPV", "ZDC" ] # a list of detectors that serve as input for the trigger processor CTP --> these need to be processed together for now ctp_trigger_inputlist = [ "FT0", "FV0", "EMC" ] @@ -693,7 +713,7 @@ def getDPL_global_options(bigshm=False, ccdbbackend=True): + ' -o ' + signalprefix + ' ' + embeddinto \ + ('', ' --timestamp ' + str(args.timestamp))[args.timestamp!=-1] + ' --run ' + str(args.run) \ + ' --vertexMode kCCDB' - if not "all" in activeDetectors: + if not isActive('all'): SGNtask['cmd'] += ' --readoutDetectors ' + " ".join(activeDetectors) if args.pregenCollContext == True: SGNtask['cmd'] += ' --fromCollContext collisioncontext.root' @@ -839,9 +859,9 @@ def putConfigValuesNew(listOfMainKeys=[], localCF = {}): TPCDigitask=createTask(name='tpcdigi_'+str(tf), needs=tpcdigineeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI"], cpu=NWORKERS, mem=str(tpcdigimem)) TPCDigitask['cmd'] = ('','ln -nfs ../bkg_HitsTPC.root . ;')[doembedding] - TPCDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ - + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + ' --tpc-lanes ' + str(NWORKERS) \ - + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini --early-forward-policy always ' \ + TPCDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + + ' --onlyDet TPC --TPCuseCCDB --interactionRate ' + str(INTRATE) + ' --tpc-lanes ' + str(NWORKERS) \ + + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini --early-forward-policy always --forceSelectedDets ' \ + putConfigValuesNew(["TPCGasParam","TPCGEMParam","TPCEleParam","TPCITCorr","TPCDetParam"], localCF={"DigiParams.maxOrbitsToDigitize" : str(orbitsPerTF), "DigiParams.seed" : str(TFSEED)}) TPCDigitask['cmd'] += (' --tpc-chunked-writer','')[args.no_tpc_digitchunking] @@ -859,10 +879,9 @@ def putConfigValuesNew(listOfMainKeys=[], localCF = {}): TRDDigitask['cmd'] = ('','ln -nfs ../bkg_HitsTRD.root . ;')[doembedding] TRDDigitask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + ' --onlyDet TRD --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' \ - + putConfigValuesNew(localCF={"TRDSimParams.digithreads" : NWORKERS, "DigiParams.seed" : str(TFSEED)}) + + putConfigValuesNew(localCF={"TRDSimParams.digithreads" : NWORKERS, "DigiParams.seed" : str(TFSEED)}) + " --forceSelectedDets" TRDDigitask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] - if isActive("TRD"): - workflow['stages'].append(TRDDigitask) + workflow['stages'].append(TRDDigitask) # these are digitizers which are single threaded def createRestDigiTask(name, det='ALLSMALLER'): @@ -882,31 +901,25 @@ def createRestDigiTask(name, det='ALLSMALLER'): tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') t['cmd'] = ('','ln -nfs ../bkg_Hits*.root . ;')[doembedding] detlist = '' - for d in smallsensorlist: - if isActive(d): - if len(detlist) > 0: - detlist += ',' - detlist += d + detlist = ','.join(smallsensorlist) t['cmd'] += commondigicmd + ' --onlyDet ' + detlist - t['cmd'] += ' --ccdb-tof-sa ' + t['cmd'] += ' --ccdb-tof-sa --forceSelectedDets ' t['cmd'] += (' --combine-devices ','')[args.no_combine_dpl_devices] t['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(t) return t else: # here we create individual digitizers - if isActive(det): - if usebkgcache: - tneeds += [ BKG_HITDOWNLOADER_TASKS[det]['name'] ] - t = createTask(name=name, needs=tneeds, - tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') - t['cmd'] = ('','ln -nfs ../bkg_Hits' + str(det) + '.root . ;')[doembedding] - t['cmd'] += commondigicmd + ' --onlyDet ' + str(det) - t['cmd'] += ('',' --disable-mc')[args.no_mc_labels] - if det == 'TOF': - t['cmd'] += ' --ccdb-tof-sa' - workflow['stages'].append(t) - return t + if usebkgcache: + tneeds += [ BKG_HITDOWNLOADER_TASKS[det]['name'] ] + t = createTask(name=name, needs=tneeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') + t['cmd'] = ('','ln -nfs ../bkg_Hits' + str(det) + '.root . ;')[doembedding] + t['cmd'] += commondigicmd + ' --onlyDet ' + str(det) + t['cmd'] += ('',' --disable-mc')[args.no_mc_labels] + if det == 'TOF': + t['cmd'] += ' --ccdb-tof-sa' + workflow['stages'].append(t) + return t det_to_digitask={} @@ -923,18 +936,19 @@ def createRestDigiTask(name, det='ALLSMALLER'): tneeds = [ContextTask['name']] if includeQED: tneeds += [QED_task['name']] - t = createTask(name="ft0fv0emcctp_digi_" + str(tf), needs=tneeds, + FT0FV0EMCCTPDIGItask = createTask(name="ft0fv0emcctp_digi_" + str(tf), needs=tneeds, tf=tf, cwd=timeframeworkdir, lab=["DIGI","SMALLDIGI"], cpu='1') - t['cmd'] = ('','ln -nfs ../bkg_HitsFT0.root . ; ln -nfs ../bkg_HitsFV0.root . ;')[doembedding] - t['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + FT0FV0EMCCTPDIGItask['cmd'] = ('','ln -nfs ../bkg_HitsFT0.root . ; ln -nfs ../bkg_HitsFV0.root . ;')[doembedding] + FT0FV0EMCCTPDIGItask['cmd'] += '${O2_ROOT}/bin/o2-sim-digitizer-workflow ' + getDPL_global_options() + ' -n ' + str(args.ns) + simsoption \ + ' --onlyDet FT0,FV0,EMC,CTP --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) \ + ' --disable-write-ini' + putConfigValuesNew(localCF={"DigiParams.seed" : str(TFSEED)}) \ - + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs - workflow['stages'].append(t) - det_to_digitask["FT0"]=t - det_to_digitask["FV0"]=t - det_to_digitask["EMC"]=t - det_to_digitask["CTP"]=t + + (' --combine-devices','')[args.no_combine_dpl_devices] + ('',' --disable-mc')[args.no_mc_labels] + QEDdigiargs \ + + ' --forceSelectedDets' + workflow['stages'].append(FT0FV0EMCCTPDIGItask) + det_to_digitask["FT0"]=FT0FV0EMCCTPDIGItask + det_to_digitask["FV0"]=FT0FV0EMCCTPDIGItask + det_to_digitask["EMC"]=FT0FV0EMCCTPDIGItask + det_to_digitask["CTP"]=FT0FV0EMCCTPDIGItask def getDigiTaskName(det): t = det_to_digitask.get(det) @@ -1007,6 +1021,7 @@ def getDigiTaskName(det): # FIXME This is so far a workaround to avoud a race condition for trdcalibratedtracklets.root TRDTRACKINGtask2 = createTask(name='trdreco2_'+str(tf), needs=[TRDTRACKINGtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem='2000') + trd_track_sources = anchorConfig.get('o2-trd-global-tracking-options', {}).get('track-sources', 'TPC,ITS-TPC') TRDTRACKINGtask2['cmd'] = '${O2_ROOT}/bin/o2-trd-global-tracking ' + getDPL_global_options(bigshm=True) + ('',' --disable-mc')[args.no_mc_labels] \ + putConfigValuesNew(['ITSClustererParam', 'ITSCATrackerParam', @@ -1015,7 +1030,7 @@ def getDigiTaskName(det): 'ft0tag', 'TPCGasParam', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) \ - + " --track-sources " + anchorConfig.get("o2-trd-global-tracking-options",{}).get("track-sources","all") \ + + " --track-sources " + trd_track_sources \ + tpc_corr_scaling_options workflow['stages'].append(TRDTRACKINGtask2) @@ -1023,11 +1038,9 @@ def getDigiTaskName(det): TOFRECOtask['cmd'] = '${O2_ROOT}/bin/o2-tof-reco-workflow --use-ccdb ' + getDPL_global_options() + putConfigValuesNew() + ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(TOFRECOtask) - toftpcmatchneeds = [TOFRECOtask['name'], TPCRECOtask['name']] - toftracksrcdefault = "TPC,ITS-TPC" - if isActive('TRD'): - toftpcmatchneeds.append(TRDTRACKINGtask2['name']) - toftracksrcdefault+=",TPC-TRD,ITS-TPC-TRD" + + toftpcmatchneeds = [TOFRECOtask['name'], TPCRECOtask['name'], ITSTPCMATCHtask['name'], TRDTRACKINGtask2['name']] + toftracksrcdefault = anchorConfig.get('o2-tof-matcher-workflow-options', {}).get('track-sources', 'TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD') TOFTPCMATCHERtask = createTask(name='toftpcmatch_'+str(tf), needs=toftpcmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') TOFTPCMATCHERtask['cmd'] = '${O2_ROOT}/bin/o2-tof-matcher-workflow ' + getDPL_global_options() \ + putConfigValuesNew(["ITSClustererParam", @@ -1037,7 +1050,7 @@ def getDigiTaskName(det): 'MFTClustererParam', 'GPU_rec_tpc', 'trackTuneParams']) \ - + " --track-sources " + anchorConfig.get("o2-tof-matcher-workflow-options",{}).get("track-sources",toftracksrcdefault) + (' --combine-devices','')[args.no_combine_dpl_devices] \ + + " --track-sources " + toftracksrcdefault + (' --combine-devices','')[args.no_combine_dpl_devices] \ + tpc_corr_scaling_options workflow['stages'].append(TOFTPCMATCHERtask) @@ -1100,11 +1113,10 @@ def getDigiTaskName(det): CPVRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(CPVRECOtask) - if args.with_ZDC: - ZDCRECOtask = createTask(name='zdcreco_'+str(tf), needs=[getDigiTaskName("ZDC")], tf=tf, cwd=timeframeworkdir, lab=["RECO", "ZDC"]) - ZDCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-zdc-digits-reco ' + getDPL_global_options() + putConfigValues() - ZDCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] - workflow['stages'].append(ZDCRECOtask) + ZDCRECOtask = createTask(name='zdcreco_'+str(tf), needs=[getDigiTaskName("ZDC")], tf=tf, cwd=timeframeworkdir, lab=["RECO", "ZDC"]) + ZDCRECOtask['cmd'] = '${O2_ROOT}/bin/o2-zdc-digits-reco ' + getDPL_global_options() + putConfigValues() + ZDCRECOtask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] + workflow['stages'].append(ZDCRECOtask) ## forward matching MCHMIDMATCHtask = createTask(name='mchmidMatch_'+str(tf), needs=[MCHRECOtask['name'], MIDRECOtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500') @@ -1130,56 +1142,16 @@ def getDigiTaskName(det): HMPRECOtask['cmd'] = '${O2_ROOT}/bin/o2-hmpid-digits-to-clusters-workflow ' + getDPL_global_options(ccdbbackend=False) + putConfigValuesNew() workflow['stages'].append(HMPRECOtask) - HMPMATCHtask = createTask(name='hmpmatch_'+str(tf), needs=[HMPRECOtask['name'],ITSTPCMATCHtask['name'],TOFTPCMATCHERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') - HMPMATCHtask['cmd'] = '${O2_ROOT}/bin/o2-hmpid-matcher-workflow ' + getDPL_global_options() + putConfigValuesNew() + hmpmatchneeds = [HMPRECOtask['name'], ITSTPCMATCHtask['name'], TOFTPCMATCHERtask['name'], TRDTRACKINGtask2['name']] + hmp_match_sources = anchorConfig.get('o2-hmpid-matcher-workflow', {}).get('track-sources', 'ITS-TPC,ITS-TPC-TRD,TPC-TRD') + HMPMATCHtask = createTask(name='hmpmatch_'+str(tf), needs=hmpmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') + HMPMATCHtask['cmd'] = '${O2_ROOT}/bin/o2-hmpid-matcher-workflow --track-sources ' + hmp_match_sources + getDPL_global_options() + putConfigValuesNew() workflow['stages'].append(HMPMATCHtask) # Take None as default, we only add more if nothing from anchorConfig - pvfinder_sources = anchorConfig.get("o2-primary-vertexing-workflow-options",{}).get("vertexing-sources", None) - pvfinder_matching_sources = anchorConfig.get("o2-primary-vertexing-workflow-options",{}).get("vertex-track-matching-sources", None) - pvfinderneeds = [ITSTPCMATCHtask['name']] - if not pvfinder_sources: - pvfinder_sources = "ITS,ITS-TPC,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" - if not pvfinder_matching_sources: - pvfinder_matching_sources = "ITS,MFT,TPC,ITS-TPC,MCH,MFT-MCH,TPC-TOF,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" - if isActive("MID"): - pvfinder_matching_sources += ",MID" - pvfinderneeds += [MIDRECOtask['name']] - if isActive('MCH') and isActive('MID'): - pvfinder_matching_sources += ",MCH-MID" - pvfinderneeds += [MCHMIDMATCHtask['name']] - - if isActive('FT0'): - pvfinderneeds += [FT0RECOtask['name']] - pvfinder_matching_sources += ",FT0" - if isActive('FV0'): - pvfinderneeds += [FV0RECOtask['name']] - pvfinder_matching_sources += ",FV0" - if isActive('FDD'): - pvfinderneeds += [FT0RECOtask['name']] - pvfinder_matching_sources += ",FDD" - if isActive('EMC'): - pvfinderneeds += [EMCRECOtask['name']] - pvfinder_matching_sources += ",EMC" - if isActive('PHS'): - pvfinderneeds += [PHSRECOtask['name']] - pvfinder_matching_sources += ",PHS" - if isActive('CPV'): - pvfinderneeds += [CPVRECOtask['name']] - pvfinder_matching_sources += ",CPV" - if isActive('TOF'): - pvfinderneeds += [TOFTPCMATCHERtask['name']] - if isActive('MFT'): - pvfinderneeds += [MFTRECOtask['name']] - if isActive('MCH'): - pvfinderneeds += [MCHRECOtask['name']] - if isActive('TRD'): - pvfinderneeds += [TRDTRACKINGtask2['name']] - if isActive('FDD'): - pvfinderneeds += [FDDRECOtask['name']] - if isActive('MFT') and isActive('MCH'): - pvfinderneeds += [MFTMCHMATCHtask['name']] - + pvfinder_sources = anchorConfig.get('o2-primary-vertexing-workflow-options', {}).get('vertexing-sources', 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP') + pvfinder_matching_sources = anchorConfig.get('', {}).get('vertex-track-matching-sources', 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP') + pvfinderneeds = [TRDTRACKINGtask2['name'], FT0RECOtask['name'], FV0RECOtask['name'], EMCRECOtask['name'], PHSRECOtask['name'], CPVRECOtask['name'], FDDRECOtask['name'], ZDCRECOtask['name'], HMPMATCHtask['name'], HMPMATCHtask['name'], ITSTPCMATCHtask['name'], TOFTPCMATCHERtask['name'], MFTMCHMATCHtask['name'], MCHMIDMATCHtask['name']] PVFINDERtask = createTask(name='pvfinder_'+str(tf), needs=pvfinderneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=NWORKERS, mem='4000') PVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-primary-vertexing-workflow ' \ @@ -1337,15 +1309,12 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if COLTYPE == "PbPb" or (doembedding and COLTYPEBKG == "PbPb"): svfinder_threads = ' --threads 8 ' svfinder_cpu = 8 + SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name'], FT0FV0EMCCTPDIGItask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000') SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000') SVFINDERtask['cmd'] = '${O2_ROOT}/bin/o2-secondary-vertexing-workflow ' SVFINDERtask['cmd'] += getDPL_global_options(bigshm=True) + svfinder_threads + putConfigValuesNew(['svertexer', 'TPCCorrMap'], {"NameConf.mDirMatLUT" : ".."}) # Take None as default, we only add more if nothing from anchorConfig - svfinder_sources = anchorConfig.get("o2-secondary-vertexing-workflow-options",{}).get("vertexing-sources", None) - if not svfinder_sources: - svfinder_sources = "ITS,TPC,ITS-TPC,TPC-TRD,TPC-TOF,ITS-TPC-TRD,ITS-TPC-TOF,ITS-TPC-TRD-TOF" - if isActive("MID"): - svfinder_sources += ",MID" + svfinder_sources = anchorConfig.get('o2-secondary-vertexing-workflow-options', {}). get('vertexing-sources', 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP') SVFINDERtask['cmd'] += ' --vertexing-sources ' + svfinder_sources + (' --combine-source-devices','')[args.no_combine_dpl_devices] # strangeness tracking is now called from the secondary vertexer if not args.with_strangeness_tracking: @@ -1359,39 +1328,9 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # produce AOD # ----------- # TODO This needs further refinement, sources and dependencies should be constructed dynamically - aodinfosources = 'ITS,MFT,MCH,TPC,ITS-TPC,MFT-MCH,ITS-TPC-TOF,TPC-TOF,FT0,FDD,TPC-TRD,ITS-TPC-TRD,ITS-TPC-TRD-TOF' + aodinfosources = anchorConfig.get('o2-aod-producer-workflow-options', {}).get('info-sources', 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP') aodneeds = [PVFINDERtask['name'], SVFINDERtask['name']] - if isActive('CTP'): - aodinfosources += ',CTP' - if isActive('FV0'): - aodneeds += [ FV0RECOtask['name'] ] - aodinfosources += ',FV0' - if isActive('TOF'): - aodneeds += [ TOFRECOtask['name'] ] - if isActive('TRD'): - aodneeds += [ TRDTRACKINGtask2['name'] ] - if isActive('EMC'): - aodneeds += [ EMCRECOtask['name'] ] - aodinfosources += ',EMC' - if isActive('CPV'): - aodneeds += [ CPVRECOtask['name'] ] - aodinfosources += ',CPV' - if isActive('PHS'): - aodneeds += [ PHSRECOtask['name'] ] - aodinfosources += ',PHS' - if isActive('MID'): - aodneeds += [ MIDRECOtask['name'] ] - aodinfosources += ',MID' - if isActive('MID') and isActive('MCH'): - aodneeds += [ MCHMIDMATCHtask['name'] ] - aodinfosources += ',MCH-MID' - if isActive('HMP'): - aodneeds += [ HMPMATCHtask['name'] ] - aodinfosources += ',HMP' - if args.with_ZDC and isActive('ZDC'): - aodneeds += [ ZDCRECOtask['name'] ] - aodinfosources += ',ZDC' if usebkgcache: aodneeds += [ BKG_KINEDOWNLOADER_TASK['name'] ] @@ -1407,7 +1346,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): if args.run_anchored == False: AODtask['cmd'] += ' --aod-timeframe-id ${ALIEN_PROC_ID}' + aod_df_id AODtask['cmd'] += ' ' + getDPL_global_options(bigshm=True) - AODtask['cmd'] += ' --info-sources ' + anchorConfig.get("o2-aod-producer-workflow-options",{}).get("info-sources",str(aodinfosources)) + AODtask['cmd'] += ' --info-sources ' + aodinfosources AODtask['cmd'] += ' --lpmp-prod-tag ${ALIEN_JDL_LPMPRODUCTIONTAG:-unknown}' AODtask['cmd'] += ' --anchor-pass ${ALIEN_JDL_LPMANCHORPASSNAME:-unknown}' AODtask['cmd'] += ' --anchor-prod ${ALIEN_JDL_MCANCHOR:-unknown}' diff --git a/MC/bin/o2dpg_workflow_utils.py b/MC/bin/o2dpg_workflow_utils.py index a029e8dee..748129de2 100755 --- a/MC/bin/o2dpg_workflow_utils.py +++ b/MC/bin/o2dpg_workflow_utils.py @@ -6,6 +6,7 @@ # List of active detectors ACTIVE_DETECTORS = ["all"] +INACTIVE_DETECTORS = [] def activate_detector(det): try: @@ -16,8 +17,11 @@ def activate_detector(det): pass ACTIVE_DETECTORS.append(det) +def deactivate_detector(det): + INACTIVE_DETECTORS.append(det) + def isActive(det): - return "all" in ACTIVE_DETECTORS or det in ACTIVE_DETECTORS + return det not in INACTIVE_DETECTORS and ("all" in ACTIVE_DETECTORS or det in ACTIVE_DETECTORS) def relativeCPU(n_rel, n_workers): # compute number of CPUs from a given number of workers diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index d0beecc0a..b5d0d80fb 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -79,6 +79,7 @@ done # the only two where there is a real default for export ALIEN_JDL_CPULIMIT=${ALIEN_JDL_CPULIMIT:-${CPULIMIT:-8}} export ALIEN_JDL_SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE:-TGeant4}} +export ALIEN_JDL_WORKFLOWDETECTORS=${ALIEN_JDL_WORKFLOWDETECTORS:-ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP} # all others MUST be set by the user/on the outside export ALIEN_JDL_LPMANCHORPASSNAME=${ALIEN_JDL_LPMANCHORPASSNAME:-${ANCHORPASSNAME}} export ALIEN_JDL_MCANCHOR=${ALIEN_JDL_MCANCHOR:-${MCANCHOR}} @@ -96,10 +97,6 @@ ALIEN_JDL_LPMPRODUCTIONTAG_KEEP=$ALIEN_JDL_LPMPRODUCTIONTAG echo "Substituting ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMPRODUCTIONTAG with ALIEN_JDL_LPMANCHORPRODUCTION=$ALIEN_JDL_LPMANCHORPRODUCTION for simulating reco pass..." ALIEN_JDL_LPMPRODUCTIONTAG=$ALIEN_JDL_LPMANCHORPRODUCTION -# ZDC causes issues for sim -#export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,ZDC,CTP -export ALIEN_JDL_WORKFLOWDETECTORS=ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP - # check variables that need to be set [ -z "${ALIEN_JDL_LPMANCHORPASSNAME}" ] && { echo "ERROR: Set ALIEN_JDL_LPMANCHORPASSNAME or ANCHORPASSNAME" ; exit 1 ; } [ -z "${ALIEN_JDL_MCANCHOR}" ] && { echo "ERROR: Set ALIEN_JDL_MCANCHOR or MCANCHOR" ; exit 1 ; } diff --git a/UTILS/parse-async-WorkflowConfig.py b/UTILS/parse-async-WorkflowConfig.py index 6252e6c65..15e713e8b 100755 --- a/UTILS/parse-async-WorkflowConfig.py +++ b/UTILS/parse-async-WorkflowConfig.py @@ -195,6 +195,13 @@ def parse_important_DPL_args(cmds, flat_config): corrstring += ' --corrma-lumi-mode ' + s2 # these are some options applied in multiple places (so save them flatly under tpc-corr-scaling) flat_config['tpc-corr-scaling'] = corrstring + + # hmp matching + if cmd == 'o2-hmpid-matcher-workflow': + c = {} + c['track-sources'] = extract_args(tokens, '--track-sources') + flat_config['o2-hmpid-matcher-workflow'] = c + def print_untreated_args(cmds): """ From 62782db57c1b0718d4f44453ab48b3f9899c9acb Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Wed, 14 Feb 2024 09:21:03 +0100 Subject: [PATCH 0360/1239] [RelVal] Add specific QC objects to compare (#1458) * put files with include patterns at RelVal/config/QC/async/_include_patterns_default.txt * add pattern files for ITS, TPC, TOF * allow comments starting with "#" Co-authored-by: Benedikt Volkel --- .../QC/async/ITS_include_patterns_default.txt | 15 +++++++++++++++ .../QC/async/TOF_include_patterns_defautl.txt | 11 +++++++++++ .../QC/async/TPC_include_patterns_default.txt | 13 +++++++++++++ RelVal/utils/o2dpg_release_validation_utils.py | 8 +++++++- 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 RelVal/config/QC/async/ITS_include_patterns_default.txt create mode 100644 RelVal/config/QC/async/TOF_include_patterns_defautl.txt create mode 100644 RelVal/config/QC/async/TPC_include_patterns_default.txt diff --git a/RelVal/config/QC/async/ITS_include_patterns_default.txt b/RelVal/config/QC/async/ITS_include_patterns_default.txt new file mode 100644 index 000000000..e63098e87 --- /dev/null +++ b/RelVal/config/QC/async/ITS_include_patterns_default.txt @@ -0,0 +1,15 @@ +# for data and MC +ITS_Tracks_PhiDistribution$ +ITS_Tracks_NClusters$ +ITS_Tracks_VertexZ$ +ITS_Tracks_VertexRvsZ$ +ITS_Tracks_VertexCoordinates$ +ITS_Clusters_Layer0_AverageClusterSizeSummary$ +ITS_Clusters_Layer3_AverageClusterSizeSummary$ +# only MC +ITS_TracksMc_efficiency_eta_ratioFromTEfficiency$ +ITS_TracksMc_efficiency_phi_ratioFromTEfficiency$ +ITS_TracksMc_efficiency_pt_ratioFromTEfficiency$ +ITS_TracksMc_faketrack_eta_ratioFromTEfficiency$ +ITS_TracksMc_faketrack_phi_ratioFromTEfficiency$ +ITS_TracksMc_faketrack_pt_ratioFromTEfficiency$ diff --git a/RelVal/config/QC/async/TOF_include_patterns_defautl.txt b/RelVal/config/QC/async/TOF_include_patterns_defautl.txt new file mode 100644 index 000000000..2a9204335 --- /dev/null +++ b/RelVal/config/QC/async/TOF_include_patterns_defautl.txt @@ -0,0 +1,11 @@ +# note that all "/" have to be replaced by "_" +TOF_Digits_DecodingErrors$ +TOF_Digits_Multiplicity_Integrated$ +TOF_MatchTrAll_mTOFChi2ITSTPC-ITSTPCTRD$ +TOF_MatchTrAll_mTOFChi2TPC$ +TOF_MatchTrAll_mTOFChi2TPCTRD$ +TOF_MatchTrAll_mEffPt_ITSTPC-ITSTPCTRD_ratioFromTEfficiency$ +TOF_PID_EvTimeTOF$ +TOF_PID_DeltaBCTOFFT0$ +TOF_PID_DeltatPi_Pt_ITSTPC_t0TOF$ +TOF_PID_DeltatPi_Pt_ITSTPCTRD_t0TOF$ diff --git a/RelVal/config/QC/async/TPC_include_patterns_default.txt b/RelVal/config/QC/async/TPC_include_patterns_default.txt new file mode 100644 index 000000000..4472729e5 --- /dev/null +++ b/RelVal/config/QC/async/TPC_include_patterns_default.txt @@ -0,0 +1,13 @@ +# note that all "/" have to be replaced by "_" +TPC_Tracks_hPhiAside$ +TPC_Tracks_hPhiCside$ +TPC_Tracks_hEta$ +TPC_Tracks_hNClustersAfterCuts$ +TPC_Tracks_hQOverPt$ +TPC_Tracks_hDCAr_A_Pos$ +TPC_Tracks_hDCAr_C_Pos$ +TPC_Tracks_hDCArVsEtaPos$ +TPC_Tracks_h2DNClustersPhiAside$ +TPC_Tracks_h2DNClustersPhiCside$ +TPC_PID_hdEdxTotMIP_TPC$ +TPC_PID_hdEdxTotVsP_TPC$ diff --git a/RelVal/utils/o2dpg_release_validation_utils.py b/RelVal/utils/o2dpg_release_validation_utils.py index 0482569aa..2607ab69e 100755 --- a/RelVal/utils/o2dpg_release_validation_utils.py +++ b/RelVal/utils/o2dpg_release_validation_utils.py @@ -282,9 +282,15 @@ def load_this_patterns(patterns): return patterns patterns_from_file = [] - with open(patterns[0][1:], "r") as f: + filename = patterns[0][1:] + if not exists(filename): + print(f"WARNING: Pattern file {filename} does not exist, not extracting any patterns!") + return + with open(filename, "r") as f: for line in f: line = line.strip() + # remove all comments; allows for inline comments or entire comment lines), then take the first token + line = line.split("#")[0].strip() if not line: continue patterns_from_file.append(line) From f751289ebb6644012a7c531ddf12ccd4925ed39c Mon Sep 17 00:00:00 2001 From: jian Date: Mon, 5 Feb 2024 22:55:03 +0100 Subject: [PATCH 0361/1239] moving MC QC from test ccdb to qcdb --- MC/bin/o2dpg_qc_finalization_workflow.py | 9 +++++---- MC/bin/o2dpg_sim_workflow.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 1b2866944..373989f20 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -33,7 +33,7 @@ def QC_finalize_name(name): return name + "_finalize" qcdir = "QC" -def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB): +def include_all_QC_finalization(ntimeframes, standalone, run, productionTag, conditionDB, qcdbHost): stages = [] @@ -49,7 +49,7 @@ def add_QC_finalization(taskName, qcConfigPath, needs=None): task = createTask(name=QC_finalize_name(taskName), needs=needs, cwd=qcdir, lab=["QC"], cpu=1, mem='2000') task['cmd'] = f'o2-qc --config {qcConfigPath} --remote-batch {taskName}.root' + \ - f' --override-values "qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag};qc.config.conditionDB.url={conditionDB}"' + \ + f' --override-values "qc.config.database.host={qcdbHost};qc.config.Activity.number={run};qc.config.Activity.periodName={productionTag};qc.config.conditionDB.url={conditionDB}"' + \ ' ' + getDPL_global_options() stages.append(task) @@ -67,7 +67,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif overrideValues = '--override-values "' overrideValues += f'qc.config.Activity.number={run};' if runSpecific else 'qc.config.Activity.number=0;' overrideValues += f'qc.config.Activity.periodName={productionTag};' if prodSpecific else 'qc.config.Activity.periodName=;' - overrideValues += f'qc.config.conditionDB.url={conditionDB}"' + overrideValues += f'qc.config.database.host={qcdbHost};qc.config.conditionDB.url={conditionDB}"' task['cmd'] = f'o2-qc --config {qcConfigPath} ' + \ overrideValues + ' ' + getDPL_global_options() stages.append(task) @@ -119,6 +119,7 @@ def main() -> int: parser.add_argument('-run',help="Run number for this MC", default=300000) parser.add_argument('-productionTag',help="Production tag for this MC", default='unknown') parser.add_argument('-conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') + parser.add_argument('-qcdbHost',help="QCDB url for QC object uploading", default='http://ali-qcdbmc-gpn.cern.ch:8083') args = parser.parse_args() print (args) @@ -140,7 +141,7 @@ def main() -> int: mkdir(qcdir) workflow={} - workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB) + workflow['stages'] = include_all_QC_finalization(ntimeframes=1, standalone=True, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost) dump_workflow(workflow["stages"], args.o) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 15f54c3bd..5b236027e 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -49,6 +49,7 @@ # - we can also sample it ourselfs here parser.add_argument('--timestamp', type=int, help="Anchoring timestamp (defaults to now)", default=-1) parser.add_argument('--conditionDB',help="CCDB url for QC workflows", default='http://alice-ccdb.cern.ch') +parser.add_argument('--qcdbHost',help="QCDB url for QC object uploading", default='http://ali-qcdbmc-gpn.cern.ch:8083') parser.add_argument('--condition-not-after', type=int, help="only consider CCDB objects not created after this timestamp (for TimeMachine)", default=3385078236000) parser.add_argument('--orbitsPerTF', type=int, help="Timeframe size in number of LHC orbits", default=128) parser.add_argument('--anchor-config',help="JSON file to contextualise workflow with external configs (config values etc.) for instance comping from data reco workflows.", default='') @@ -1168,7 +1169,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # the --local-batch argument will make QC Tasks store their results in a file and merge with any existing objects task['cmd'] = f'{readerCommand} | o2-qc --config {configFilePath}' + \ f' --local-batch ../{qcdir}/{objectsFile}' + \ - f' --override-values "qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \ + f' --override-values "qc.config.database.host={args.qcdbHost};qc.config.Activity.number={args.run};qc.config.Activity.periodName={args.productionTag};qc.config.Activity.start={args.timestamp};qc.config.conditionDB.url={args.conditionDB}"' + \ ' ' + getDPL_global_options(ccdbbackend=False) # Prevents this task from being run for multiple TimeFrames at the same time, thus trying to modify the same file. task['semaphore'] = objectsFile @@ -1423,7 +1424,7 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): job_merging = False if includeFullQC: - workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB)) + workflow['stages'].extend(include_all_QC_finalization(ntimeframes=NTIMEFRAMES, standalone=False, run=args.run, productionTag=args.productionTag, conditionDB=args.conditionDB, qcdbHost=args.qcdbHost)) if includeAnalysis: From 4171dcffd87c9de94d627b2769e3f38c5602bdbf Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Tue, 13 Feb 2024 15:11:34 +0100 Subject: [PATCH 0362/1239] JDL vars don't allow spaces or underscore: only 1 word --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index bc65f87fb..d178ca5db 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -321,7 +321,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then fi DISABLE_CORRECTIONS= - [[ -n "$ALIEN_JDL_MSHAPE_CORRECTION" && $ALIEN_JDL_MSHAPE_CORRECTION == "0" ]] && ENABLE_MSHAPE=0 || ENABLE_MSHAPE=1 + [[ -n "$ALIEN_JDL_MSHAPECORRECTION" && $ALIEN_JDL_MSHAPECORRECTION == "0" ]] && ENABLE_MSHAPE=0 || ENABLE_MSHAPE=1 if [[ -n $MEAN_IR_FOR_TPC ]] ; then # firs check if corrections were not disabled via MEAN_IR_FOR_TPC if [[ $MEAN_IR_FOR_TPC -gt 0 ]] ; then # positive value overrides map mean lumi @@ -352,7 +352,7 @@ elif [[ $ALIGNLEVEL == 1 ]]; then elif [[ $INST_IR_FOR_TPC == "CTPCCDB" ]]; then # using what we have in the CCDB CTP counters, extracted at the beginning of the script echo "Using CTP CCDB which gave the mean IR of the run at the beginning of the script ($RUN_IR Hz)" export TPC_CORR_SCALING+=";TPCCorrMap.lumiInst=$RUN_IR" - else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INST_IR_FOR_TPC = $ALIEN_JDL_INST_IR_FOR_TPC)" + else echo "Unknown setting for INST_IR_FOR_TPC = $INST_IR_FOR_TPC (with ALIEN_JDL_INSTIRFORTPC = $ALIEN_JDL_INSTIRFORTPC)" return 1 fi From c8c56d7c6551f73672f197e935d55f4ad57f67c7 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Wed, 14 Feb 2024 15:46:29 +0100 Subject: [PATCH 0363/1239] [RelVal] Fix typo in filename (#1459) Co-authored-by: Benedikt Volkel --- ...lude_patterns_defautl.txt => TOF_include_patterns_default.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename RelVal/config/QC/async/{TOF_include_patterns_defautl.txt => TOF_include_patterns_default.txt} (100%) diff --git a/RelVal/config/QC/async/TOF_include_patterns_defautl.txt b/RelVal/config/QC/async/TOF_include_patterns_default.txt similarity index 100% rename from RelVal/config/QC/async/TOF_include_patterns_defautl.txt rename to RelVal/config/QC/async/TOF_include_patterns_default.txt From b7950acf91dd64500ff647717940a1c927e6ce1d Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Wed, 14 Feb 2024 16:51:34 +0100 Subject: [PATCH 0364/1239] [RelVal] Add MFT default objects, add comment to ITS file (#1461) Co-authored-by: Benedikt Volkel --- RelVal/config/QC/async/ITS_include_patterns_default.txt | 1 + RelVal/config/QC/async/MFT_include_patterns_default.txt | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 RelVal/config/QC/async/MFT_include_patterns_default.txt diff --git a/RelVal/config/QC/async/ITS_include_patterns_default.txt b/RelVal/config/QC/async/ITS_include_patterns_default.txt index e63098e87..f8a391236 100644 --- a/RelVal/config/QC/async/ITS_include_patterns_default.txt +++ b/RelVal/config/QC/async/ITS_include_patterns_default.txt @@ -1,3 +1,4 @@ +# note that all "/" have to be replaced by "_" # for data and MC ITS_Tracks_PhiDistribution$ ITS_Tracks_NClusters$ diff --git a/RelVal/config/QC/async/MFT_include_patterns_default.txt b/RelVal/config/QC/async/MFT_include_patterns_default.txt new file mode 100644 index 000000000..a2b013ea1 --- /dev/null +++ b/RelVal/config/QC/async/MFT_include_patterns_default.txt @@ -0,0 +1,9 @@ +# note that all "/" have to be replaced by "_" +MFT_Tracks_mMFTTrackEta_5_MinClusters$ +MFT_Tracks_mMFTTrackPhi_5_MinClusters$ +MFT_Tracks_mMFTTrackEta_6_MinClusters$ +MFT_Tracks_mMFTTrackPhi_6_MinClusters$ +MFT_Tracks_mMFTTrackInvQPt$ +MFT_Tracks_mMFTTrackNumberOfClusters$ +MFT_Tracks_mMFTTrackROFSize$ +MFT_Clusters_mClustersROFSize$ From 843a10020b04fa8462ee73ecc371d74a97242f82 Mon Sep 17 00:00:00 2001 From: catalinristea Date: Thu, 15 Feb 2024 09:09:12 +0200 Subject: [PATCH 0365/1239] Update anchorMC.sh - added proc arg (#1462) * Update anchorMC.sh - added proc arg --- MC/run/ANCHOR/anchorMC.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index b5d0d80fb..438a790b8 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -191,7 +191,7 @@ ALICEO2_CCDB_LOCALCACHE=${ALICEO2_CCDB_LOCALCACHE:-$(pwd)/ccdb} baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}" # these arguments will be passed as well but only evetually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py -remainingargs="-gen pythia8 -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" +remainingargs="-gen pythia8 -proc inel -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" remainingargs="${remainingargs} -e ${ALIEN_JDL_SIMENGINE} -j ${NWORKERS}" remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" remainingargs="${remainingargs} --anchor-config config-json.json" From a4f859c3f48682f75487ce8fca7d8530e3fae1fe Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 15 Feb 2024 21:27:38 +0100 Subject: [PATCH 0366/1239] Debug mode for topology generation (#1460) --- DATA/tools/epn/gen_topo.sh | 6 +++++- DATA/tools/epn/gen_topo_o2dpg.sh | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/DATA/tools/epn/gen_topo.sh b/DATA/tools/epn/gen_topo.sh index 973dd64b5..297dddd2f 100755 --- a/DATA/tools/epn/gen_topo.sh +++ b/DATA/tools/epn/gen_topo.sh @@ -18,7 +18,11 @@ if [[ -z "$EPN2EOS_METAFILES_DIR" ]] && [[ "0$WORKFLOWMODE" != "0print" ]]; then export EPN2EOS_METAFILES_DIR=/data/epn2eos_tool/epn2eos # Directory for epn2eos meta data files fi if [[ $USER == "epn" ]]; then - [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=/scratch/services/gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. + if [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=/scratch/services/staging_gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. + else + [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=/scratch/services/gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. + fi else [[ -z "$GEN_TOPO_WORKDIR" ]] && export GEN_TOPO_WORKDIR=$HOME/gen_topo # Working directory for checkout of O2DPG repository and for XML cache. If this directory is wiped, gen_topo will recreate all necessary content the next time it runs. The folder should be persistent to cache workflows. mkdir -p $HOME/gen_topo diff --git a/DATA/tools/epn/gen_topo_o2dpg.sh b/DATA/tools/epn/gen_topo_o2dpg.sh index 29346796d..3da6f701e 100755 --- a/DATA/tools/epn/gen_topo_o2dpg.sh +++ b/DATA/tools/epn/gen_topo_o2dpg.sh @@ -22,6 +22,15 @@ if [[ -z "$MULTIPLICITY_FACTOR_REST" ]]; then echo \$MULTIPLICITY_FACTOR_REST mi if [[ -z "$RECOSHMSIZE" ]]; then echo \$RECOSHMSIZE missing; exit 1; fi # SHM Size for reconstruction collections if [[ -z "$DDSHMSIZE" ]]; then echo \$DDSHMSIZE missing; exit 1; fi # SHM Size for DD +# In case of debug mode, overwrite some settings +if [[ "${DEBUG_TOPOLOGY_GENERATION:=0}" == "1" ]]; then + echo "Debugging mode enabled. Setting options accordingly" 1>&2 + RECO_NUM_NODES_OVERRIDE=1 # to avoid slurm query, specify number of nodes to fixed value + GEN_TOPO_MI100_NODES=1 # also for MI100 nodes + GEN_TOPO_OVERRIDE_TEMPDIR=$PWD # keep temporary files like QC jsons in local directory + EPN2EOS_METAFILES_DIR=/tmp # nothing is written here, just needs to be set to something +fi + # Check settings coming from the EPN if [[ -z "$FILEWORKDIR" ]]; then echo \$FILEWORKDIR missing; exit 1; fi if [[ -z "$INRAWCHANNAME" ]]; then echo \$INRAWCHANNAME missing; exit 1; fi @@ -103,6 +112,7 @@ while true; do break done + if [[ ! -z "$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD" ]] && [[ "0$WORKFLOWMODE" != "0print" ]]; then TMP_POST_CACHING_CMD="$GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_CMD $GEN_TOPO_ODC_EPN_TOPO_POST_CACHING_ARGS" TMP_POST_CACHING_NMIN=$(( $RECO_NUM_NODES_OVERRIDE > $RECO_MAX_FAIL_NODES_OVERRIDE ? $RECO_NUM_NODES_OVERRIDE - $RECO_MAX_FAIL_NODES_OVERRIDE : 0 )) @@ -126,6 +136,9 @@ if [[ ! -z "$ECS_ENVIRONMENT_ID" && -d "/var/log/topology/" && $USER == "epn" ]] fi cat $GEN_TOPO_WORKDIR/output.xml -echo Removing temporary output file $GEN_TOPO_WORKDIR/output.xml 1>&2 -rm $GEN_TOPO_WORKDIR/output.xml + +if [[ "$DEBUG_TOPOLOGY_GENERATION" == "0" ]]; then + echo Removing temporary output file $GEN_TOPO_WORKDIR/output.xml 1>&2 + rm $GEN_TOPO_WORKDIR/output.xml +fi rm -f $GEN_TOPO_LOCKFILE From bda1a376702e52ad891b2490aad7a52973f23024 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 16 Feb 2024 10:13:02 +0100 Subject: [PATCH 0367/1239] Revert "Update anchorMC.sh - added proc arg (#1462)" This reverts commit 843a10020b04fa8462ee73ecc371d74a97242f82. --- MC/run/ANCHOR/anchorMC.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 438a790b8..b5d0d80fb 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -191,7 +191,7 @@ ALICEO2_CCDB_LOCALCACHE=${ALICEO2_CCDB_LOCALCACHE:-$(pwd)/ccdb} baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}" # these arguments will be passed as well but only evetually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py -remainingargs="-gen pythia8 -proc inel -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" +remainingargs="-gen pythia8 -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" remainingargs="${remainingargs} -e ${ALIEN_JDL_SIMENGINE} -j ${NWORKERS}" remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" remainingargs="${remainingargs} --anchor-config config-json.json" From e13c468a0dc52e06396dd8e4ac43337ad3c59f4d Mon Sep 17 00:00:00 2001 From: shahoian Date: Fri, 16 Feb 2024 17:17:42 +0100 Subject: [PATCH 0368/1239] adjust matching chi2 cut to 100 and max c14 diff to 2.5 --- DATA/production/configurations/asyncReco/setenv_extra.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index d178ca5db..4f9992a67 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -293,8 +293,8 @@ elif [[ $ALIGNLEVEL == 1 ]]; then ERROB="100e-8" [[ -z $TPCITSTIMEERR ]] && TPCITSTIMEERR="0.2" [[ -z $ITS_CONFIG || "$ITS_CONFIG" != *"--tracking-mode"* ]] && export ITS_CONFIG+=" --tracking-mode async" - CUT_MATCH_CHI2=80 - export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.XMatchingRef=60.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=1.5;tpcitsMatch.crudeNSigma2Cut[0]=64;tpcitsMatch.crudeNSigma2Cut[1]=64;tpcitsMatch.crudeNSigma2Cut[2]=64;tpcitsMatch.crudeNSigma2Cut[3]=64;tpcitsMatch.crudeNSigma2Cut[4]=64;" + CUT_MATCH_CHI2=100 + export ITSTPCMATCH="tpcitsMatch.safeMarginTimeCorrErr=2.;tpcitsMatch.XMatchingRef=60.;tpcitsMatch.cutMatchingChi2=$CUT_MATCH_CHI2;;tpcitsMatch.crudeAbsDiffCut[0]=6;tpcitsMatch.crudeAbsDiffCut[1]=6;tpcitsMatch.crudeAbsDiffCut[2]=0.3;tpcitsMatch.crudeAbsDiffCut[3]=0.3;tpcitsMatch.crudeAbsDiffCut[4]=2.5;tpcitsMatch.crudeNSigma2Cut[0]=64;tpcitsMatch.crudeNSigma2Cut[1]=64;tpcitsMatch.crudeNSigma2Cut[2]=64;tpcitsMatch.crudeNSigma2Cut[3]=64;tpcitsMatch.crudeNSigma2Cut[4]=64;" #-------------------------------------- TPC corrections ----------------------------------------------- # we need to provide to TPC From 598afb3d09044292eeecdbd95d67a93c384f10ca Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 5 Feb 2024 16:46:31 +0100 Subject: [PATCH 0369/1239] Fix type error in pipeline runner --- MC/bin/o2_dpg_workflow_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index 44f56303a..b8cd05bc0 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -36,7 +36,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-f','--workflowfile', help='Input workflow file name', required=True) -parser.add_argument('-jmax','--maxjobs', help='Number of maximal parallel tasks.', default=100) +parser.add_argument('-jmax','--maxjobs', type=int, help='Number of maximal parallel tasks.', default=100) parser.add_argument('-k','--keep-going', action='store_true', help='Keep executing the pipeline as far possibe (not stopping on first failure)') parser.add_argument('--dry-run', action='store_true', help='Show what you would do.') parser.add_argument('--visualize-workflow', action='store_true', help='Saves a graph visualization of workflow.') @@ -1730,5 +1730,5 @@ def speedup_ROOT_Init(): exit(code) actionlogger.info("Running in cgroup") -executor=WorkflowExecutor(args.workflowfile,jmax=args.maxjobs,args=args) +executor=WorkflowExecutor(args.workflowfile,jmax=int(args.maxjobs),args=args) exit (executor.execute()) From e996e47d70442af1400dec323dbef7d1f6bf1aaf Mon Sep 17 00:00:00 2001 From: swenzel Date: Mon, 19 Feb 2024 09:18:36 +0100 Subject: [PATCH 0370/1239] pipeline_runner: Fix script creation following a recent change in class data layout introduced here: https://github.com/AliceO2Group/O2DPG/commit/ec4acee8f0d38616b6fa45809661213c9b938acc --- MC/bin/o2_dpg_workflow_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index b8cd05bc0..3c826476a 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -1488,8 +1488,8 @@ def produce_script(self, filename): # we record the global environment setting # in particular to capture global workflow initialization lines.append('#-- GLOBAL INIT SECTION FROM WORKFLOW --\n') - for e in self.globalenv: - lines.append('export ' + str(e) + '=' + str(self.globalenv[e]) + '\n') + for e in self.globalinit['env']: + lines.append('export ' + str(e) + '=' + str(self.globalinit['env'][e]) + '\n') lines.append('#-- TASKS FROM WORKFLOW --\n') for tid in taskorder: print ('Doing task ' + self.idtotask[tid]) From a883150b1806f5006add0b5523cabb25455df6ca Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 19 Feb 2024 10:15:06 +0100 Subject: [PATCH 0371/1239] [AnchorMC] Allow for any additional option for sim WF (#1463) Use as ALIEN_JDL_ANCHOR_SIM_OPTIONS="-ini -confKey "key=value;otherKey=otherValue" -trigger -productionTag myTag" Build the final remainingargs by prepending it. The last argument wins, hence in this case -productionTag would not be overwritten with this user choice Co-authored-by: Benedikt Volkel --- MC/bin/o2dpg_sim_workflow_anchored.py | 3 ++- MC/run/ANCHOR/anchorMC.sh | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index a50a0532a..9b859e9f0 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -409,7 +409,7 @@ def main(): effTrigger = 28.0 # this is ZDC else: effTrigger = 0.759 - + # time needs to be converted to seconds ==> timestamp / 1000 rate = retrieve_MinBias_CTPScaler_Rate(ctp_scalers, timestamp/1000., effTrigger, grplhcif.getBunchFilling().getNBunches(), ColSystem) @@ -424,6 +424,7 @@ def main(): # we finally pass forward to the unanchored MC workflow creation # TODO: this needs to be done in a pythonic way clearly + # NOTE: forwardargs can - in principle - contain some of the arguments that are appended here. However, the last passed argument wins, so they would be overwritten. forwardargs += " -tf " + str(args.tf) + " --sor " + str(sor) + " --timestamp " + str(timestamp) + " --production-offset " + str(prod_offset) + " -run " + str(args.run_number) + " --run-anchored --first-orbit " \ + str(first_orbit) + " -field ccdb -bcPatternFile ccdb" + " --orbitsPerTF " + str(GLOparams["OrbitsPerTF"]) + " -col " + str(ColSystem) + " -eCM " + str(eCM) + ' --readoutDets ' + GLOparams['detList'] print ("forward args ", forwardargs) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index b5d0d80fb..b5c0422c3 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -76,10 +76,12 @@ done # Allow for both "ALIEN_JDL_LPM" as well as "KEY" -# the only two where there is a real default for +# the only four where there is a real default for export ALIEN_JDL_CPULIMIT=${ALIEN_JDL_CPULIMIT:-${CPULIMIT:-8}} export ALIEN_JDL_SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE:-TGeant4}} export ALIEN_JDL_WORKFLOWDETECTORS=${ALIEN_JDL_WORKFLOWDETECTORS:-ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP} +# can be passed to contain additional options that will be passed to o2dpg_sim_workflow_anchored.py and eventually to o2dpg_sim_workflow.py +export ALIEN_JDL_ANCHOR_SIM_OPTIONS=${ALIEN_JDL_ANCHOR_SIM_OPTIONS:--gen pythia8} # all others MUST be set by the user/on the outside export ALIEN_JDL_LPMANCHORPASSNAME=${ALIEN_JDL_LPMANCHORPASSNAME:-${ANCHORPASSNAME}} export ALIEN_JDL_MCANCHOR=${ALIEN_JDL_MCANCHOR:-${MCANCHOR}} @@ -191,10 +193,12 @@ ALICEO2_CCDB_LOCALCACHE=${ALICEO2_CCDB_LOCALCACHE:-$(pwd)/ccdb} baseargs="-tf ${NTIMEFRAMES} --split-id ${SPLITID} --prod-split ${PRODSPLIT} --cycle ${CYCLE} --run-number ${ALIEN_JDL_LPMRUNNUMBER}" # these arguments will be passed as well but only evetually be digested by o2dpg_sim_workflow.py which is called from o2dpg_sim_workflow_anchored.py -remainingargs="-gen pythia8 -seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" +remainingargs="-seed ${SEED} -ns ${NSIGEVENTS} --include-local-qc --pregenCollContext" remainingargs="${remainingargs} -e ${ALIEN_JDL_SIMENGINE} -j ${NWORKERS}" remainingargs="${remainingargs} -productionTag ${ALIEN_JDL_LPMPRODUCTIONTAG:-alibi_anchorTest_tmp}" -remainingargs="${remainingargs} --anchor-config config-json.json" +# prepend(!) ALIEN_JDL_ANCHOR_SIM_OPTIONS +# since the last passed argument wins, e.g. -productionTag cannot be overwritten by the user +remainingargs="${ALIEN_JDL_ANCHOR_SIM_OPTIONS} ${remainingargs} --anchor-config config-json.json" echo "baseargs: ${baseargs}" echo "remainingargs: ${remainingargs}" From f721817f954fa477b145175122188b42dede1cb4 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 19 Feb 2024 11:56:37 +0100 Subject: [PATCH 0372/1239] [SimCI] Do NOT run everything if anything changed in test directory (#1475) --- test/run_generator_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_generator_tests.sh b/test/run_generator_tests.sh index bb256ba78..34efd5443 100755 --- a/test/run_generator_tests.sh +++ b/test/run_generator_tests.sh @@ -378,7 +378,7 @@ pushd ${REPO_DIR} > /dev/null # First check, if testing itself has changed. In that case this will add INI files # for which a test can be found -global_testing_changed=$(get_changed_files | grep -E ".C$|.sh$" | grep "^test/") +global_testing_changed=$(get_changed_files | grep -E "common/kine_tests/test_generic_kine.C|run_generator_tests.sh" | grep "^test/") [[ "${global_testing_changed}" != "" ]] && add_ini_files_from_all_tests # Then add the ini files that have changed as well. We need to do that so we get information From 26319e42c27e7f5c15a17d0b351ebbfc1f18d403 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 19 Feb 2024 10:09:28 +0100 Subject: [PATCH 0373/1239] dpl-workflow: Change default FLP processing in sync mode to CTP only --- DATA/common/setenv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index f7342e553..b4e5152eb 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -134,7 +134,7 @@ else # Defaults when running on the EPN if [[ -z "${SHMTHROW:-}" ]]; then export SHMTHROW=0; fi if [[ -z "${TIMEFRAME_SHM_LIMIT:-}" ]]; then export TIMEFRAME_SHM_LIMIT=$(( $SHMSIZE / 2 )); fi if [[ -z "${EDJSONS_DIR:-}" ]]; then export EDJSONS_DIR="/scratch/services/ed/jsons_${RUNTYPE}"; fi - if [[ -z "${WORKFLOW_DETECTORS_FLP_PROCESSING+x}" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING="TOF,CTP"; fi # Current default in sync processing is that FLP processing is only enabled for TOF + if [[ -z "${WORKFLOW_DETECTORS_FLP_PROCESSING+x}" ]]; then export WORKFLOW_DETECTORS_FLP_PROCESSING="CTP"; fi # Current default in sync processing is that FLP processing is only enabled for TOF if [[ -z "${GEN_TOPO_AUTOSCALE_PROCESSES:-}" ]]; then export GEN_TOPO_AUTOSCALE_PROCESSES=1; fi # On the EPN we should make sure to always use the node to the full extent fi # Some more options for running on the EPN From d3e61262e4ec8eaf2f4cedd077976f5d6f414704 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 20 Feb 2024 10:16:39 +0100 Subject: [PATCH 0374/1239] [SimWF] Small fixes (#1476) * o2dpg_sim_workflow_anchored.py * remove unused function from CCDBAccessor due to undefined utility function that is used inside it * fix variable name * o2dpg_sim_workflow.py * fix dependency of cleanup task * o2dpg-workflow-tools.py * use return value of function correctly Co-authored-by: Benedikt Volkel --- MC/bin/o2dpg-workflow-tools.py | 7 ++++--- MC/bin/o2dpg_sim_workflow.py | 6 +++--- MC/bin/o2dpg_sim_workflow_anchored.py | 12 +----------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/MC/bin/o2dpg-workflow-tools.py b/MC/bin/o2dpg-workflow-tools.py index 78b82294e..09bd4b115 100755 --- a/MC/bin/o2dpg-workflow-tools.py +++ b/MC/bin/o2dpg-workflow-tools.py @@ -15,15 +15,16 @@ def extend(args): is kept """ # load workflows - workflow_orig = read_workflow(args.orig_wf) - workflow_extend = read_workflow(args.extend_wf) + workflow_orig, meta = read_workflow(args.orig_wf) + workflow_extend, _ = read_workflow(args.extend_wf) # extend workflow_orig.extend(workflow_extend) # dump in new file filename = args.output if args.output else args.orig_wf - dump_workflow(workflow_orig, filename) + # propagate meta information from original workflow that is extended + dump_workflow(workflow_orig, filename, meta) def create(args): diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 5b236027e..d0812d42c 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1039,7 +1039,7 @@ def getDigiTaskName(det): TOFRECOtask['cmd'] = '${O2_ROOT}/bin/o2-tof-reco-workflow --use-ccdb ' + getDPL_global_options() + putConfigValuesNew() + ('',' --disable-mc')[args.no_mc_labels] workflow['stages'].append(TOFRECOtask) - + toftpcmatchneeds = [TOFRECOtask['name'], TPCRECOtask['name'], ITSTPCMATCHtask['name'], TRDTRACKINGtask2['name']] toftracksrcdefault = anchorConfig.get('o2-tof-matcher-workflow-options', {}).get('track-sources', 'TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD') TOFTPCMATCHERtask = createTask(name='toftpcmatch_'+str(tf), needs=toftpcmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') @@ -1407,10 +1407,10 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): # taking away digits, clusters and other stuff as soon as possible. # TODO: cleanup by labels or task names if args.early_tf_cleanup == True: - TFcleanup = createTask(name='tfcleanup_'+str(tf), needs= [ AOD_merge_task['name'] ], tf=tf, cwd=timeframeworkdir, lab=["CLEANUP"], mem='0', cpu='1') + TFcleanup = createTask(name='tfcleanup_'+str(tf), needs= [ AODtask['name'] ], tf=tf, cwd=timeframeworkdir, lab=["CLEANUP"], mem='0', cpu='1') TFcleanup['cmd'] = 'rm *digi*.root;' TFcleanup['cmd'] += 'rm *cluster*.root' - workflow['stages'].append(TFcleanup); + workflow['stages'].append(TFcleanup) # AOD merging as one global final step aodmergerneeds = ['aod_' + str(tf) for tf in range(1, NTIMEFRAMES + 1)] diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 9b859e9f0..9708e2bc3 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -50,16 +50,6 @@ def __init__(self, url): # we allow nullptr responsens and will treat it ourselves o2.ccdb.BasicCCDBManager.instance().setFatalWhenNull(False) - def list(self, path, dump_path=None): - ret = self.api.list(path, False, "application/json") - ret = json.loads(ret) - if ret and "objects" in ret: - ret = ret["objects"] - if ret and dump_path: - print(f"CCDB object information for path {path} stored in {dump_path}") - dump_json(ret, dump_path) - return ret - def fetch(self, path, obj_type, timestamp=None, meta_info=None): """ TODO We could use CcdbApi::snapshot at some point, needs revision @@ -95,7 +85,7 @@ def retrieve_sor_eor(ccdbreader, run_number): path_run_info = "RCT/Info/RunInformation" header = ccdbreader.fetch_header(path_run_info, run_number) if not header: - print(f"WARNING: Cannot find run information for run number {r}") + print(f"WARNING: Cannot find run information for run number {run_number}") return None # return this a dictionary return {"SOR": int(header["SOR"]), "EOR": int(header["EOR"])} From 2c5d4f3271f066dd0a7e0050b38ccf8ac855b460 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 10:29:53 +0100 Subject: [PATCH 0375/1239] Fix syntax checks for shell scripts (#1473) --- DATA/common/gen_topo_helper_functions.sh | 14 +++++++------- DATA/common/setenv_calib.sh | 2 +- .../2022/LHC22f/apass1/setenv_extra.sh | 1 + .../configurations/asyncReco/async_pass.sh | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/DATA/common/gen_topo_helper_functions.sh b/DATA/common/gen_topo_helper_functions.sh index 00fc1f118..6efe2d7f8 100755 --- a/DATA/common/gen_topo_helper_functions.sh +++ b/DATA/common/gen_topo_helper_functions.sh @@ -104,37 +104,37 @@ _check_multiple() has_detectors() { - _check_multiple has_detector $@ + _check_multiple has_detector "$@" } has_detectors_qc() { - _check_multiple has_detector_qc $@ + _check_multiple has_detector_qc "$@" } has_detectors_calib() { - _check_multiple has_detector_calib $@ + _check_multiple has_detector_calib "$@" } has_detectors_reco() { - _check_multiple has_detector_reco $@ + _check_multiple has_detector_reco "$@" } has_detectors_ctf() { - _check_multiple has_detector_ctf $@ + _check_multiple has_detector_ctf "$@" } has_detectors_flp_processing() { - _check_multiple has_detector_flp_processing $@ + _check_multiple has_detector_flp_processing "$@" } workflow_has_parameters() { - _check_multiple workflow_has_parameter $@ + _check_multiple workflow_has_parameter "$@" } add_comma_separated() diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index a1d2e7692..0b44fe23c 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -14,7 +14,7 @@ SOURCE_GUARD_SETENV_CALIB=1 # define the conditions for each calibration if has_detector_calib ITS && has_detectors_reco ITS && has_detector_matching PRIMVTX && [[ ! -z "$VERTEXING_SOURCES" ]]; then CAN_DO_CALIB_PRIMVTX_MEANVTX=1; else CAN_DO_CALIB_PRIMVTX_MEANVTX=0; fi if has_detector_calib TOF && has_detector_reco TOF; then CAN_DO_CALIB_TOF_DIAGNOSTICS=1; CAN_DO_CALIB_TOF_INTEGRATEDCURR=1; else CAN_DO_CALIB_TOF_DIAGNOSTICS=0; CAN_DO_CALIB_TOF_INTEGRATEDCURR=0; fi -if has_detector_calib TOF && has_detector_reco TOF && (( has_detectors_reco ITS TPC && has_detector_matching ITSTPCTOF ) || ( has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRDTOF )); then CAN_DO_CALIB_TOF_LHCPHASE=1; CAN_DO_CALIB_TOF_CHANNELOFFSETS=1; else CAN_DO_CALIB_TOF_LHCPHASE=0; CAN_DO_CALIB_TOF_CHANNELOFFSETS=0; fi +if has_detector_calib TOF && has_detector_reco TOF && ( ( has_detectors_reco ITS TPC && has_detector_matching ITSTPCTOF ) || ( has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRDTOF ) ); then CAN_DO_CALIB_TOF_LHCPHASE=1; CAN_DO_CALIB_TOF_CHANNELOFFSETS=1; else CAN_DO_CALIB_TOF_LHCPHASE=0; CAN_DO_CALIB_TOF_CHANNELOFFSETS=0; fi if has_detector_calib TPC && has_detectors ITS TPC TOF TRD && has_detector_matching ITSTPCTRDTOF; then CAN_DO_CALIB_TPC_SCDCALIB=1; else CAN_DO_CALIB_TPC_SCDCALIB=0; fi if has_detector_calib TPC && has_processing_step TPC_DEDX; then CAN_DO_CALIB_TPC_TIMEGAIN=1; CAN_DO_CALIB_TPC_RESPADGAIN=1; else CAN_DO_CALIB_TPC_TIMEGAIN=0; CAN_DO_CALIB_TPC_RESPADGAIN=0; fi if has_detector_calib TPC && has_detectors ITS TPC && has_detector_matching ITSTPC; then CAN_DO_CALIB_TPC_VDRIFTTGL=1; else CAN_DO_CALIB_TPC_VDRIFTTGL=0; fi diff --git a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh index 648e9a61b..e814f833a 100644 --- a/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/setenv_extra.sh @@ -72,6 +72,7 @@ fi CTP_BC_SHIFT=0 if [[ $ALIEN_JDL_LPMANCHORYEAR == "2022" ]]; then CTP_BC_SHIFT=-294 +fi if [[ $RUNNUMBER -ge 538923 ]] && [[ $RUNNUMBER -le 539700 ]]; then # 3 BC offset (future direction) in CTP data observed for LHC23zd - LHC23zs CTP_BC_SHIFT=-3 diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 05ed0dea2..9d1f49025 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -576,7 +576,7 @@ else echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" > validation_error.message echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" > validation_error.message echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" > validation_error.message - exit 1000 + exit 255 fi fi fi @@ -745,7 +745,7 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then CURRENT_POOL_SIZE=`jobs -r | wc -l` done < $JOB_LIST # collecting return codes of the merging processes - for i in ${!arr[@]}; do + for i in "${!arr[@]}"; do wait ${arr[$i]} exitcode=$? if [[ $exitcode -ne 0 ]]; then From 5452aba0aaaf8683defeb19a39e334bab8adcdce Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 10:30:16 +0100 Subject: [PATCH 0376/1239] TPC SCD calib send track data by default (#1474) --- DATA/common/setenv_calib.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 0b44fe23c..bdb836509 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -36,8 +36,9 @@ if [[ $SYNCMODE != 1 ]] && has_detector_reco TPC && has_detector_reco ITS && has # additional individual settings for calibration workflows if has_detector CTP; then export CALIB_TPC_SCDCALIB_CTP_INPUT="--enable-ctp"; else export CALIB_TPC_SCDCALIB_CTP_INPUT=""; fi if [[ ${DISABLE_TRD_PH:-} == 1 ]]; then CAN_DO_CALIB_TRD_T0=0; fi -# the slot length needs to be known both on the aggregator and the processing nodes, therefore it is defined (in seconds!) here -: ${CALIB_TPC_SCDCALIB_SLOTLENGTH:=600} + +: ${CALIB_TPC_SCDCALIB_SLOTLENGTH:=600} # the slot length needs to be known both on the aggregator and the processing nodes, therefore it is defined (in seconds!) here +: ${CALIB_TPC_SCDCALIB_SENDTRKDATA:=1} # by default, we want to write the track information in addition to unbinned residuals to allow finer filtering offline if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then From d24070e8be0a4e7180825f40402eee5f21e45e53 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 10:31:26 +0100 Subject: [PATCH 0377/1239] For synthetic runs we set IS_SIMULATED_DATA=1 (#1477) --- DATA/common/setenv.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index b4e5152eb..74ae5640e 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -104,6 +104,7 @@ if [[ -z "${RAWINPUTDIR:-}" ]]; then export RAWINPUTDIR=$FILEWORKDIR; fi # if [[ -z "${EPNSYNCMODE:-}" ]]; then export EPNSYNCMODE=0; fi # Is this workflow supposed to run on EPN for sync processing? Will enable InfoLogger / metrics / fetching QC JSONs from consul... if [[ -z "${BEAMTYPE:-}" ]]; then export BEAMTYPE=PbPb; fi # Beam type, must be PbPb, pp, pPb, cosmic, technical if [[ -z "${RUNTYPE:-}" ]]; then export RUNTYPE=Standalone; fi # Run Type, standalone for local tests, otherwise PHYSICS, COSMICS, TECHNICAL, SYNTHETIC +if [[ $RUNTYPE == "SYNTHETIC" ]]; then export IS_SIMULATED_DATA=1; fi # For SYNTHETIC runs we always process simulated data if [[ -z "${IS_SIMULATED_DATA:-}" ]]; then export IS_SIMULATED_DATA=1; fi # processing simulated data if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi # processing triggered data (TPC triggered instead of continuous) if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs From 7f48808ee5ee611abfbe0f01b363cf1954ca22c2 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 20 Feb 2024 10:40:41 +0100 Subject: [PATCH 0378/1239] Add tool to fetch PR information based on assigned labels (#1478) * applicable to different repos * distinguishes between merged and other (simply closed or still open) PRs * dumps the output into a simple text file for further proessing Co-authored-by: Benedikt Volkel --- UTILS/o2dpg_make_github_pr_report.py | 141 +++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100755 UTILS/o2dpg_make_github_pr_report.py diff --git a/UTILS/o2dpg_make_github_pr_report.py b/UTILS/o2dpg_make_github_pr_report.py new file mode 100755 index 000000000..b6a4ac9c9 --- /dev/null +++ b/UTILS/o2dpg_make_github_pr_report.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# Get list of PRs from provided repo that have a certain label assigned +# Can be used to figure out which PRs should be ported + +import sys +import argparse +import requests + + +def organise_prs(prs): + """ + Sort PRs by time merged, starting from old to recent + """ + # collect merged PRs + prs_merged = [] + # collect the time of merged PRs + merged_at = [] + # other PRs, open, closed and not merged + prs_other = [] + + for pr in prs: + if not pr['merged_at']: + # that has not been merged + prs_other.append(pr) + continue + # get the PR itself and the merged timestamp + prs_merged.append(pr) + merged_at.append(pr['merged_at']) + + # sort the merged PRs by their merged timestamp + prs_merged = [pr for _, pr in sorted(zip(merged_at, prs))] + + return prs_merged, prs_other + + +def get_prs(owner, repo, prod_label, pr_state, include_unmerged, per_page=50, start_page=1, pages=1): + """ + Get PRs according to some selection + """ + # GitHub API endpoint for listing closed pull requests with a specific label + merged_token = '&is:merged=true' if not include_unmerged else '' + prs_return = [] + + has_error = False + for page in range(start_page, pages + 1): + url = f'https://api.github.com/repos/{owner}/{repo}/pulls?state={pr_state}{merged_token}&page={page}&per_page={per_page}' + print(f'Fetch PRs accrodring to {url}') + + # Send GET request to GitHub API + response = requests.get(url) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse JSON response + prs = response.json() + # PRs to return because we filter on a specific label + for pr in prs: + labels = pr['labels'] + accept = False + for label in labels: + if label['name'] == prod_label: + # only with the correct the label will be accepted + accept = True + break + if not accept: + continue + # we will end up here if accepted, so append + prs_return.append(pr) + + else: + print(f'Failed to retrieve data: {response.status_code} - {response.text}') + has_error = True + break + + if has_error: + return None, None + + # organise PRs into different lists (merged and others) + return organise_prs(prs_return) + + +def make_report(prs_merged, prs_other, outfile): + """ + Make a report + + simply dump into text file + """ + + with open(outfile, 'w') as f: + f.write('# FROM OLDEST TO RECENT\n') + # our common header + f.write('| Date of next tag | Requestor | Package | PR | Data or MC | Comment | JIRA (if it exists) | Accepted | In production | Validated by requestor |\n') + f.write('| ---------------- | ------------ | ------- | --------------------------------------------------------:|:--------------------------------------------- | ------------------- | ---------------- | ------------- |-------------| ------------------|\n') + + # first put the merged PRs + for pr in prs_merged: + mc_data = [] + + for label in pr['labels']: + if label['name'] in ('MC', 'DATA'): + # get assigned MC or DATA label if this PR has it + mc_data.append(label['name']) + + # if no specific MC or DATA label, assume valid for both + mc_data = ','.join(mc_data) if mc_data else 'MC,DATA' + # add the full line to the output file + f.write(f'| {args.date} | {pr["user"]["login"]} | {args.repo} | [PR]({pr["html_url"]}) | {mc_data} | {pr["title"]} | | | | |\n') + + # add all the other commits + f.write('OTHER PRs\n') + for pr in prs_other: + f.write(f'| {args.date} | {pr["user"]["login"]} | {args.repo} | [PR]({pr["html_url"]}) | | {pr["title"]} | | | | |\n') + + +if __name__ == '__main__': + # Parse command-line arguments + parser = argparse.ArgumentParser(description='Retrieve closed pull requests with a specific label from a GitHub repository') + parser.add_argument('--owner', help='GitHub repository owner', default='AliceO2Group') + parser.add_argument('--repo', required=True, help='GitHub repository name, e.g. O2DPG or AliceO2') + parser.add_argument('--prod-label', dest='prod_label', required=True, help='Production label to filter PRs') + parser.add_argument('--pr-state', dest='pr_state', default='closed', help='The state of the PR') + parser.add_argument('--include-unmerged', dest='include_unmerged', action='store_true', help='To fetch also unmerged PRs') + parser.add_argument('--output', default='o2dpg_pr_report.txt') + parser.add_argument('--date', help='The date tag to be put', required=True) + parser.add_argument('--per-page', dest='per_page', default=50, help='How many results per page') + parser.add_argument('--start-page', dest='start_page', type=int, default=1, help='Start on this page') + parser.add_argument('--pages', type=int, default=1, help='Number of pages') + + + args = parser.parse_args() + + # Retrieve closed pull requests with the specified label + prs_merged, prs_other = get_prs(args.owner, args.repo, args.prod_label, args.pr_state, args.include_unmerged, args.per_page, args.start_page, args.pages) + if prs_merged is None: + print('ERROR: There was a problem fetching the info.') + sys.exit(1) + + make_report(prs_merged, prs_other, args.output) + + sys.exit(0) From ed7cc8640f26e1098f5ac1e467b57806400ba59c Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 20 Feb 2024 11:22:06 +0100 Subject: [PATCH 0379/1239] [Anchor] Add test for anchored MC (#1464) Runs a very simple anchored production, 2TFs, 50 pp events each Co-authored-by: Benedikt Volkel --- MC/run/ANCHOR/anchorMC.sh | 2 + .../tests/test_anchor_2023_apass2_PbPb.sh | 36 ++++++++++ .../tests/test_anchor_2023_apass2_pp.sh | 39 +++++++++++ test/common/utils/utils.sh | 2 +- test/run_workflow_tests.sh | 67 +++++++++++++++++-- 5 files changed, 139 insertions(+), 7 deletions(-) create mode 100755 MC/run/ANCHOR/tests/test_anchor_2023_apass2_PbPb.sh create mode 100755 MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index b5c0422c3..2bab586de 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -241,6 +241,8 @@ if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include # do QC tasks echo "Doing QC" ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k + # NOTE that with the -k|--keep-going option, the runner will try to keep on executing even if some tasks fail. + # That means, even if there is a failing QC task, the return code will be 0 MCRC=$? fi diff --git a/MC/run/ANCHOR/tests/test_anchor_2023_apass2_PbPb.sh b/MC/run/ANCHOR/tests/test_anchor_2023_apass2_PbPb.sh new file mode 100755 index 000000000..095908e4b --- /dev/null +++ b/MC/run/ANCHOR/tests/test_anchor_2023_apass2_PbPb.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# +# An example steering script for anchored MC simulations, PbPb +# + +# example anchoring +# taken from https://its.cern.ch/jira/browse/O2-4586 +export ALIEN_JDL_LPMANCHORPASSNAME=apass2 +export ALIEN_JDL_MCANCHOR=apass2 +export ALIEN_JDL_COLLISIONSYSTEM=Pb-Pb +export ALIEN_JDL_CPULIMIT=8 +export ALIEN_JDL_LPMPASSNAME=apass2 +export ALIEN_JDL_LPMRUNNUMBER=544121 +export ALIEN_JDL_LPMPRODUCTIONTYPE=MC +export ALIEN_JDL_LPMINTERACTIONTYPE=PbPb +export ALIEN_JDL_LPMPRODUCTIONTAG=LHC24a1 +export ALIEN_JDL_LPMANCHORRUN=544121 +export ALIEN_JDL_LPMANCHORPRODUCTION=LHC23zzh +export ALIEN_JDL_LPMANCHORYEAR=2023 + +export NTIMEFRAMES=2 +export NSIGEVENTS=2 +export SPLITID=100 +export PRODSPLIT=153 +export CYCLE=0 + +# on the GRID, this is set, for our use case, we can mimic any job ID +export ALIEN_PROC_ID=2963436952 + +# run the central anchor steering script; this includes +# * derive timestamp +# * derive interaction rate +# * extract and prepare configurations (which detectors are contained in the run etc.) +# * run the simulation (and QC) +${O2DPG_ROOT}/MC/run/ANCHOR/anchorMC.sh diff --git a/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh b/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh new file mode 100755 index 000000000..e528b8a0a --- /dev/null +++ b/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# +# An example steering script for anchored MC simulations, pp +# + +# example anchoring +# taken from https://its.cern.ch/jira/browse/O2-4586 +export ALIEN_JDL_LPMANCHORPASSNAME=apass2 +export ALIEN_JDL_MCANCHOR=apass2 +export ALIEN_JDL_COLLISIONSYSTEM=p-p +export ALIEN_JDL_CPULIMIT=8 +export ALIEN_JDL_LPMPASSNAME=apass2 +export ALIEN_JDL_LPMRUNNUMBER=535069 +export ALIEN_JDL_LPMPRODUCTIONTYPE=MC +export ALIEN_JDL_LPMINTERACTIONTYPE=pp +export ALIEN_JDL_LPMPRODUCTIONTAG=LHC24a2 +export ALIEN_JDL_LPMANCHORRUN=535069 +export ALIEN_JDL_LPMANCHORPRODUCTION=LHC23f +export ALIEN_JDL_LPMANCHORYEAR=2023 + +export NTIMEFRAMES=2 +export NSIGEVENTS=50 +export SPLITID=100 +export PRODSPLIT=153 +export CYCLE=0 + +# on the GRID, this is set, for our use case, we can mimic any job ID +export ALIEN_PROC_ID=2963436952 + +# for pp and 50 events per TF, we launch only 4 workers. +export NWORKERS=4 + +# run the central anchor steering script; this includes +# * derive timestamp +# * derive interaction rate +# * extract and prepare configurations (which detectors are contained in the run etc.) +# * run the simulation (and QC) +${O2DPG_ROOT}/MC/run/ANCHOR/anchorMC.sh diff --git a/test/common/utils/utils.sh b/test/common/utils/utils.sh index e2cae2171..0c34c5395 100644 --- a/test/common/utils/utils.sh +++ b/test/common/utils/utils.sh @@ -78,7 +78,7 @@ make_wf_creation_script() print_error_logs() { local search_dir=${1} - local search_pattern="TASK-EXIT-CODE: ([1-9][0-9]*)|[Ss]egmentation violation|[Ee]xception caught|\[FATAL\]|uncaught exception|\(int\) ([1-9][0-9]*)|fair::FatalException" + local search_pattern="TASK-EXIT-CODE: ([1-9][0-9]*)|[Ss]egmentation violation|[Ss]egmentation fault|Program crashed|[Ee]xception caught|\[FATAL\]|uncaught exception|\(int\) ([1-9][0-9]*)|fair::FatalException" local error_files=$(find ${search_dir} -maxdepth 4 -type f \( -name "*.log" -or -name "*serverlog*" -or -name "*workerlog*" -or -name "*mergerlog*" \) | xargs grep -l -E "${search_pattern}" | sort) for ef in ${error_files} ; do echo_red "Error found in log $(realpath ${ef})" diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index c4a2daeaa..adc610393 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -3,12 +3,14 @@ # The test parent dir to be cretaed in current directory TEST_PARENT_DIR_PWG="o2dpg_tests/workflows_pwgs" TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_bin" +TEST_PARENT_DIR_ANCHORED="o2dpg_tests/anchored" # a global counter for tests TEST_COUNTER=0 # unified names of log files LOG_FILE_WF="o2dpg-test-wf.log" +LOG_FILE_ANCHORED="o2dpg-test-anchored.log" # Prepare some colored output SRED="\033[0;31m" @@ -123,6 +125,26 @@ run_workflow_creation() return ${RET} } +test_anchored() +{ + local to_run="${1:-${O2DPG_ROOT}/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh}" + local RET=0 + for anchored_script in ${to_run} ; do + [[ ! -f ${anchored_script} ]] && { echo "Desired test script ${anchored_script} does not exist. Skip." ; continue ; } + ((TEST_COUNTER++)) + local test_dir=${TEST_COUNTER}_$(basename ${anchored_script})_dir + rm -rf ${test_dir} 2> /dev/null + mkdir ${test_dir} + pushd ${test_dir} > /dev/null + echo -n "Test ${TEST_COUNTER}: ${anchored_script}" + ${anchored_script} >> ${LOG_FILE_ANCHORED} 2>&1 + local ret_this=${?} + [[ "${ret_this}" != "0" ]] && RET=${ret_this} + popd > /dev/null + done + return ${RET} +} + collect_changed_pwg_wf_files() { # Collect all INI files which have changed @@ -188,8 +210,9 @@ source ${REPO_DIR}/test/common/utils/utils.sh pushd ${REPO_DIR} > /dev/null # flag if anything changed in the sim workflow bin dir -changed_wf_bin=$(get_changed_files | grep "MC/bin") +changed_wf_bin=$(get_changed_files | grep -E "MC/bin") changed_wf_bin_related=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json") +changed_anchored_related=$(get_changed_files | grep -E "MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py") # collect what has changed for PWGs @@ -215,6 +238,27 @@ REPO_DIR=$(realpath ${REPO_DIR}) export O2DPG_ROOT=${REPO_DIR} +############### +# ANCHORED MC # +############### +# prepare our local test directory for PWG tests +rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null +mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null +pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null + +# global return code for PWGs +ret_global_anchored=0 +if [[ "${changed_anchored_related}" != "" ]] ; then + echo "### Test anchored ###" + # Run an anchored test + test_anchored + ret_global_anchored=${?} + echo +fi + +# return to where we came from +popd > /dev/null + ######## # PWGs # ######## @@ -226,7 +270,7 @@ pushd ${TEST_PARENT_DIR_PWG} > /dev/null # global return code for PWGs ret_global_pwg=0 if [[ "${changed_wf_bin}" != "" ]] ; then - # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files eal=rlier + # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files earlier WF_FILES=$(get_all_workflows "MC/run/.*/") echo fi @@ -240,7 +284,6 @@ if [[ "${WF_FILES}" != "" ]] ; then echo fi - # return to where we came from popd > /dev/null @@ -285,9 +328,21 @@ if [[ "${ret_global_bin}" != "0" ]] ; then echo "###################################" echo print_error_logs ${TEST_PARENT_DIR_BIN} - exit ${ret_global_bin} fi +# However, if a central test fails, exit code will be !=0 +if [[ "${ret_global_anchored}" != "0" ]] ; then + echo + echo "##########################" + echo "# ERROR for anchored MCs #" + echo "##########################" + echo + print_error_logs ${TEST_PARENT_DIR_ANCHORED} +fi + +RET=$(( ret_global_bin + ret_global_anchored )) + echo -echo_green "All required workflow tests successful" -echo +[[ "${RET}" != "0" ]] && echo "There were errors, please check!" || echo_green "All required workflow tests successful" + +exit ${RET} From 070a0f30bf57a53b44ace67876afd7bbbe2de915 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 20 Feb 2024 12:32:08 +0100 Subject: [PATCH 0380/1239] [SimCI] Change $@ to $* (#1480) Co-authored-by: Benedikt Volkel --- test/run_analysisqc_tests.sh | 4 ++-- test/run_generator_tests.sh | 4 ++-- test/run_relval_tests.sh | 4 ++-- test/run_workflow_tests.sh | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/run_analysisqc_tests.sh b/test/run_analysisqc_tests.sh index fab8dd083..bd57493fd 100755 --- a/test/run_analysisqc_tests.sh +++ b/test/run_analysisqc_tests.sh @@ -14,13 +14,13 @@ SEND="\033[0m" echo_green() { - echo -e "${SGREEN}$@${SEND}" + echo -e "${SGREEN}${*}${SEND}" } echo_red() { - echo -e "${SRED}$@${SEND}" + echo -e "${SRED}${*}${SEND}" } diff --git a/test/run_generator_tests.sh b/test/run_generator_tests.sh index 34efd5443..d5a4d3c74 100755 --- a/test/run_generator_tests.sh +++ b/test/run_generator_tests.sh @@ -37,13 +37,13 @@ SEND="\033[0m" echo_green() { - echo -e "${SGREEN}$@${SEND}" + echo -e "${SGREEN}${*}${SEND}" } echo_red() { - echo -e "${SRED}$@${SEND}" + echo -e "${SRED}${*}${SEND}" } diff --git a/test/run_relval_tests.sh b/test/run_relval_tests.sh index 6294e1361..a14c3e0ed 100755 --- a/test/run_relval_tests.sh +++ b/test/run_relval_tests.sh @@ -14,13 +14,13 @@ SEND="\033[0m" echo_green() { - echo -e "${SGREEN}$@${SEND}" + echo -e "${SGREEN}${*}${SEND}" } echo_red() { - echo -e "${SRED}$@${SEND}" + echo -e "${SRED}${*}${SEND}" } diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index adc610393..e96b07141 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -20,13 +20,13 @@ SEND="\033[0m" echo_green() { - echo -e "${SGREEN}$@${SEND}" + echo -e "${SGREEN}${*}${SEND}" } echo_red() { - echo -e "${SRED}$@${SEND}" + echo -e "${SRED}${*}${SEND}" } get_git_repo_directory() From eb41cb7c60d1a1cf5544e4e321dbb298a7ae19ea Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 20 Feb 2024 12:40:52 +0100 Subject: [PATCH 0381/1239] [RelVa] Fix imported function name (#1481) Co-authored-by: Benedikt Volkel --- RelVal/o2dpg_overlay_plots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RelVal/o2dpg_overlay_plots.py b/RelVal/o2dpg_overlay_plots.py index 9f9a420b7..12682ca0b 100755 --- a/RelVal/o2dpg_overlay_plots.py +++ b/RelVal/o2dpg_overlay_plots.py @@ -17,7 +17,7 @@ o2dpg_release_validation = importlib.util.module_from_spec(spec) spec.loader.exec_module(o2dpg_release_validation) sys.modules["o2dpg_release_validation"] = o2dpg_release_validation -from o2dpg_release_validation import only_extract_impl +from o2dpg_release_validation import extract_and_flatten spec = importlib.util.spec_from_file_location("o2dpg_release_validation_plot", join(O2DPG_ROOT, "RelVal", "utils", 'o2dpg_release_validation_plot.py')) o2dpg_release_validation_plot = importlib.util.module_from_spec(spec) @@ -39,7 +39,7 @@ def run(args): ref_file = None for i, (input_file, label) in enumerate(zip(args.inputs, args.labels)): - _, config = only_extract_impl(input_file, args.output, label, prefix=i, reference_extracted=ref_file) + _, config = extract_and_flatten(input_file, args.output, label, prefix=i, reference_extracted=ref_file) if not config: print(f"ERROR: Problem with input file {input_file}, cannot extract") return 1 From 27b9a48382e6cd1ed4597d281fde7dcf09d82938 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 14:18:46 +0100 Subject: [PATCH 0382/1239] More bash syntax check fixes (#1479) --- DATA/production/calib/mch-badchannel-processing.sh | 1 - DATA/production/calib/tpc-pulser-long.sh | 1 - DATA/production/calib/tpc-pulser.sh | 1 - .../configurations/2021/ctf_recreation/ctf_recreation.sh | 2 +- .../configurations/2022/LHC22f/apass1/async_pass.sh | 4 ++-- 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/DATA/production/calib/mch-badchannel-processing.sh b/DATA/production/calib/mch-badchannel-processing.sh index 771aeab87..738020262 100755 --- a/DATA/production/calib/mch-badchannel-processing.sh +++ b/DATA/production/calib/mch-badchannel-processing.sh @@ -1,4 +1,3 @@ - #!/bin/bash source common/setenv.sh diff --git a/DATA/production/calib/tpc-pulser-long.sh b/DATA/production/calib/tpc-pulser-long.sh index e1aaab4d4..e7124fd28 100755 --- a/DATA/production/calib/tpc-pulser-long.sh +++ b/DATA/production/calib/tpc-pulser-long.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash source common/setenv.sh diff --git a/DATA/production/calib/tpc-pulser.sh b/DATA/production/calib/tpc-pulser.sh index 6b68a030b..6b62e559f 100755 --- a/DATA/production/calib/tpc-pulser.sh +++ b/DATA/production/calib/tpc-pulser.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash source common/setenv.sh diff --git a/DATA/production/configurations/2021/ctf_recreation/ctf_recreation.sh b/DATA/production/configurations/2021/ctf_recreation/ctf_recreation.sh index bbf361c95..be5085e9f 100755 --- a/DATA/production/configurations/2021/ctf_recreation/ctf_recreation.sh +++ b/DATA/production/configurations/2021/ctf_recreation/ctf_recreation.sh @@ -72,7 +72,7 @@ fi # if "$DETCONFIG" is set explicitly, this has the second highest priority # last option is to have it from the JDL if [[ -z "$DETCONFIG" ]]; then - if [[ -z "ALIEN_JDL_DETCONFIG" ]]; then + if [[ -z "$ALIEN_JDL_DETCONFIG" ]]; then echo "nothing set the detector configuration to use, exiting" exit 4 else diff --git a/DATA/production/configurations/2022/LHC22f/apass1/async_pass.sh b/DATA/production/configurations/2022/LHC22f/apass1/async_pass.sh index 3dc46a02c..ce764852f 100755 --- a/DATA/production/configurations/2022/LHC22f/apass1/async_pass.sh +++ b/DATA/production/configurations/2022/LHC22f/apass1/async_pass.sh @@ -504,7 +504,7 @@ else echo "nCTFsFilesInspected_step1 = $nCTFsFilesInspected_step1, nCTFsFilesInspected_step2 = $nCTFsFilesInspected_step2" > validation_error.message echo "nCTFsFilesOK_step1 = $nCTFsFilesOK_step1, nCTFsFilesOK_step2 = $nCTFsFilesOK_step2" > validation_error.message echo "nCTFsProcessed_step1 = $nCTFsProcessed_step1, nCTFsProcessed_step2 = $nCTFsProcessed_step2" > validation_error.message - exit 1000 + exit 255 fi fi fi @@ -663,7 +663,7 @@ if [[ $ALIEN_JDL_AODOFF != 1 ]]; then CURRENT_POOL_SIZE=`jobs -r | wc -l` done < $JOB_LIST # collecting return codes of the merging processes - for i in ${!arr[@]}; do + for i in "${!arr[@]}"; do wait ${arr[$i]} exitcode=$? if [[ $exitcode -ne 0 ]]; then From 6d2d239238e99971d9856682d97e5f71fea7bcfb Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Tue, 20 Feb 2024 14:22:18 +0100 Subject: [PATCH 0383/1239] Fix Bash syntax issues found by shellcheck (#1482) Fix issues found in https://github.com/AliceO2Group/O2DPG/pull/1469, in addition to those fixed in https://github.com/AliceO2Group/O2DPG/pull/1479. --- GRID/utils/extractErroredLogFiles.sh | 4 ++-- GRID/utils/grid_submit.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GRID/utils/extractErroredLogFiles.sh b/GRID/utils/extractErroredLogFiles.sh index 81c21839d..20a0750ba 100755 --- a/GRID/utils/extractErroredLogFiles.sh +++ b/GRID/utils/extractErroredLogFiles.sh @@ -4,13 +4,13 @@ # Beware that errors might occur outside of O2DPG tasks such as in preprocessing etc or not visible in logs mytar () { - tar $@ + tar "$@" } if [[ $(uname) == "Darwin" ]]; then echo "Running on macOS. This needs gnu-tar" $(which gtar) mytar () { - gtar $@ + gtar "$@" } fi diff --git a/GRID/utils/grid_submit.sh b/GRID/utils/grid_submit.sh index 7913f83ad..f364bed68 100755 --- a/GRID/utils/grid_submit.sh +++ b/GRID/utils/grid_submit.sh @@ -401,7 +401,7 @@ EOF spin[1]="|" spin[0]="\\" JOBSTATUS="I" - if [ "{WAITFORALIEN}" ]; then + if [ "${WAITFORALIEN}" ]; then echo -n "Waiting for jobs to return ... Last status : ${spin[0]} ${JOBSTATUS}" fi counter=0 From ba2ab9450929a2c14706ca87b3235fde3a285a22 Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Tue, 20 Feb 2024 16:54:14 +0100 Subject: [PATCH 0384/1239] Add syntax checkers for Bash and Python (#1469) --- .github/workflows/check-json-syntax.yml | 37 ---------- .github/workflows/syntax-checks.yml | 95 +++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 37 deletions(-) delete mode 100644 .github/workflows/check-json-syntax.yml create mode 100644 .github/workflows/syntax-checks.yml diff --git a/.github/workflows/check-json-syntax.yml b/.github/workflows/check-json-syntax.yml deleted file mode 100644 index fae51ae62..000000000 --- a/.github/workflows/check-json-syntax.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: Validate JSON syntax - -# Run on any commit or PR that changes any JSON file. -'on': - push: - paths: - - '**.json' - pull_request: - paths: - - '**.json' - -permissions: {} - -jobs: - json-syntax: - name: validate syntax - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Validate syntax for JSON files - run: | - error=0 - readarray -d '' json_files < \ - <(find . \( -path ./.git -or -path ./DATA/testing/private \) -prune -false -or -type f -name '*.json' -print0) - for jsonf in "${json_files[@]}"; do - echo "::debug::Checking $jsonf..." - if ! errmsg=$(jq . "$jsonf" 2>&1 >/dev/null); then - error=1 - echo "Invalid JSON syntax found in $jsonf:" >&2 - printf '::error file=%s,title=%s::%s\n' "$jsonf" 'Invalid JSON syntax' "$errmsg" - fi - done - exit "$error" diff --git a/.github/workflows/syntax-checks.yml b/.github/workflows/syntax-checks.yml new file mode 100644 index 000000000..54164102d --- /dev/null +++ b/.github/workflows/syntax-checks.yml @@ -0,0 +1,95 @@ +--- +name: Validate syntax + +'on': + - push + - pull_request + +permissions: {} + +jobs: + json-syntax: + name: JSON + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate syntax for JSON files + run: | + error=0 + readarray -d '' json_files < \ + <(find . \( -path ./.git -or -path ./DATA/testing/private \) -prune -false -or -type f -name '*.json' -print0) + for jsonf in "${json_files[@]}"; do + echo "::debug::Checking $jsonf..." + if ! errmsg=$(jq . "$jsonf" 2>&1 >/dev/null); then + error=1 + echo "Invalid JSON syntax found in $jsonf:" >&2 + printf '::error file=%s,title=%s::%s\n' "$jsonf" 'Invalid JSON syntax' "$errmsg" + fi + done + exit "$error" + + bash-syntax: + name: Bash + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Validate syntax with bash -n + run: | + error=0 + readarray -d '' files < \ + <(find . -path ./.git -prune -false -or -type f -name '*.sh' -print0) + for bashf in "${files[@]}"; do + echo "::debug::Checking $bashf..." + if ! errmsg=$(bash -n "$bashf" 2>&1 >/dev/null); then + error=1 + echo "Invalid Bash syntax found in $bashf:" >&2 + printf '::error file=%s,title=%s::%s\n' "$bashf" 'Invalid syntax' "$errmsg" + fi + done + exit "$error" + + shellcheck: + name: Shellcheck + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Shellcheck to find errors + run: | + error=0 + find . -path ./.git -prune -false -or -type f -name '*.sh' -print0 | + xargs -0 shellcheck -xf json1 -S error -s bash > errors.json || error=$? + # Produce code annotations in GitHub's format. + jq -r '.comments[] | "Error found in \(.file) line \(.line):\n::error file=\(.file),line=\(.line),endLine=\(.endLine),col=\(.column),endColumn=\(.endColumn)::\(.message)"' errors.json + exit "$error" + + pylint: + name: Pylint + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install prerequisites + run: | + sudo apt update -y + sudo apt install -y pylint + + - name: Run Pylint to find errors + run: | + error=0 + find . -path ./.git -prune -false -or -type f -name '*.py' -print0 | + # "import-errors" are shown for valid modules like ROOT, so ignore them. + xargs -0 pylint -E -f json --disable import-error > errors.json || error=$? + # Produce code annotations in GitHub's format. + jq -r '.[] | "Error found in \(.path) line \(.line):\n::error file=\(.path),line=\(.line),endLine=\(.endLine),col=\(.column),endColumn=\(.endColumn),title=Pylint \(.type) \(.symbol)::\(.message)"' errors.json + exit "$error" From fe36cb8f8e27faee1f39c45b3912931d0905a77c Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 16:10:48 +0100 Subject: [PATCH 0385/1239] Set beam type dependent defaults for TPC_CORR_SCALING --- DATA/common/setenv.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 74ae5640e..ae7e276f2 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -110,7 +110,6 @@ if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing -if [[ -z "${TPC_CORR_SCALING:-}" ]]; then export TPC_CORR_SCALING=""; fi # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1> and TPCCorrMap... configurable param if [[ $EPNSYNCMODE == 0 ]]; then if [[ -z "${SHMSIZE:-}" ]]; then export SHMSIZE=$(( 8 << 30 )); fi # Size of shared memory for messages if [[ -z "${NGPUS:-}" ]]; then export NGPUS=1; fi # Number of GPUs to use, data distributed round-robin @@ -164,6 +163,13 @@ DISABLE_ROOT_INPUT="--disable-root-input" : ${DISABLE_DIGIT_CLUSTER_INPUT="--clusters-from-upstream"} # Special detector related settings +if [[ -z "${TPC_CORR_SCALING:-}" ]]; then # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1> and TPCCorrMap... configurable param + TPC_CORR_SCALING= + if [[ $BEAMTYPE == "pp" ]] || [[ $BEAMTYPE == "PbPb" ]]; then TPC_CORR_SCALING+="--lumi-type 1 TPCCorrMap.lumiInstFactor=2.414"; fi + if [[ $BEAMTYPE == "cosmic" ]]; then TPC_CORR_SCALING=" TPCCorrMap.lumiMean=-1;"; fi # for COSMICS we disable all corrections + export TPC_CORR_SCALING=$TPC_CORR_SCALING +fi + MID_FEEID_MAP="$FILEWORKDIR/mid-feeId_mapper.txt" ITSMFT_STROBES="" From 7313180ebb96573bc6fc08d48f5b5e852cec702e Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 20 Feb 2024 16:11:14 +0100 Subject: [PATCH 0386/1239] ED_VERTEX_MODE has been removed --- DATA/production/workflow-multiplicities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 81db5f102..275200d4f 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -222,7 +222,7 @@ if [[ -z ${EVE_NTH_EVENT:-} ]]; then EVE_NTH_EVENT=2 elif [[ "$HIGH_RATE_PP" == "1" ]]; then EVE_NTH_EVENT=10 - elif [[ $BEAMTYPE == "pp" && "${ED_VERTEX_MODE:-}" == "1" ]]; then + elif [[ $BEAMTYPE == "pp" ]]; then EVE_NTH_EVENT=$((4 * 250 / $RECO_NUM_NODES_WORKFLOW_CMP)) else # COSMICS / TECHNICALS / ... EVE_NTH_EVENT=1 From 62aaa86dcc059b172221debf96ca8e2d1033e9b2 Mon Sep 17 00:00:00 2001 From: Mattia Faggin Date: Wed, 21 Feb 2024 20:07:14 +0100 Subject: [PATCH 0387/1239] Change eta cut for TPC tracks in GLO QC. (#1487) Co-authored-by: Mattia Faggin --- DATA/production/qc-async/itstpc.json | 2 +- MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/qc-async/itstpc.json b/DATA/production/qc-async/itstpc.json index 5bce54d2e..b294b8cf9 100644 --- a/DATA/production/qc-async/itstpc.json +++ b/DATA/production/qc-async/itstpc.json @@ -49,7 +49,7 @@ "minNITSClustersCut": "0", "maxChi2PerClusterITS": "100000", "minPtTPCCut": "0.1f", - "etaTPCCut": "1.4f", + "etaTPCCut": "0.9f", "minNTPCClustersCut": "60", "minDCACut": "100.f", "minDCACutY": "10.f", diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index a005c00de..27dff87b8 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -46,7 +46,7 @@ "minNITSClustersCut": "0", "maxChi2PerClusterITS": "100000", "minPtTPCCut": "0.1f", - "etaTPCCut": "1.4f", + "etaTPCCut": "0.9f", "minNTPCClustersCut": "60", "minDCACut": "100.f", "minDCACutY": "10.f", From 30afe4927ae7bf1ad556305aaecd1c1dde9588a4 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 22 Feb 2024 15:13:22 +0100 Subject: [PATCH 0388/1239] Enable IDC and SAC processing by default (#1485) * Enable IDC and SAC processing by default * Add missing fi --- DATA/common/setenv_calib.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index bdb836509..ae07638c6 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -76,15 +76,25 @@ if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then if [[ $CAN_DO_CALIB_TPC_VDRIFTTGL == 1 ]]; then if [[ -z ${CALIB_TPC_VDRIFTTGL+x} ]]; then CALIB_TPC_VDRIFTTGL=1; fi fi - # IDCs + # IDCs (by default we enable it for running the synch. reco on the EPNs) if [[ $CAN_DO_CALIB_TPC_IDC == 1 ]]; then - if [[ -z ${CALIB_TPC_IDC+x} ]] || [[ $CALIB_TPC_IDC == 0 ]]; then - CALIB_TPC_IDC=0; # default is off + if [[ -z ${CALIB_TPC_IDC+x} ]]; then + if [[ $EPNSYNCMODE == 1 ]]; then + CALIB_TPC_IDC=1; + else + CALIB_TPC_IDC=0; + fi fi fi - # SAC + # SAC (by default we enable it for running the synch. reco on the EPNs) if [[ $CAN_DO_CALIB_TPC_SAC == 1 ]]; then - if [[ -z ${CALIB_TPC_SAC+x} ]]; then CALIB_TPC_SAC=0; fi # default is off + if [[ -z ${CALIB_TPC_SAC+x} ]]; then + if [[ $EPNSYNCMODE == 1 ]]; then + CALIB_TPC_SAC=1; + else + CALIB_TPC_SAC=0; + fi + fi fi # calibrations for TRD From 6d4924eef63aaf6f4922b2fe2691fffbd0387fb7 Mon Sep 17 00:00:00 2001 From: fmazzasc Date: Wed, 21 Feb 2024 14:55:11 +0100 Subject: [PATCH 0389/1239] Update hypernuclei gun for pp --- MC/config/PWGLF/pythia8/generator/hypernuclei.gun | 14 +++++++------- MC/run/PWGLF/run_HyperNucleiInjectedGap.sh | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun index 55af09ba2..af21f0faf 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei.gun @@ -1,8 +1,8 @@ # PDG N ptMin ptMax yMin yMax -1000010020 1 0.2 6 -1 1 -1000010030 1 0.2 6 -1 1 -1000020030 1 0.2 6 -1 1 -1000020040 1 0.2 6 -1 1 -1010010030 1 0.2 6 -1 1 -1010010040 1 0.2 6 -1 1 -1010020040 1 0.2 6 -1 1 +1000010020 1 0.2 10 -1 1 +1000010030 1 0.2 10 -1 1 +1000020030 1 0.2 10 -1 1 +1000020040 1 0.2 10 -1 1 +1010010030 1 0.2 10 -1 1 +1010010040 1 0.2 10 -1 1 +1010020040 1 0.2 10 -1 1 diff --git a/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh b/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh index 027f6d0a0..cbf094547 100644 --- a/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh +++ b/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh @@ -15,7 +15,7 @@ NWORKERS=${NWORKERS:-8} MODULES="--skipModules ZDC" -SIMENGINE=${SIMENGINE:-TGeant3} +SIMENGINE=${SIMENGINE:-TGeant4} NSIGEVENTS=${NSIGEVENTS:-1} NBKGEVENTS=${NBKGEVENTS:-1} NTIMEFRAMES=${NTIMEFRAMES:-1} From 31573f5c3ef96868fe8ce33f1c157a390fa67657 Mon Sep 17 00:00:00 2001 From: fmazzasc Date: Tue, 20 Feb 2024 17:55:49 +0100 Subject: [PATCH 0390/1239] Update hypernuclei gun for pbpb --- MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini | 2 +- MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun diff --git a/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini b/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini index f548c57cf..feceea039 100644 --- a/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini +++ b/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini @@ -1,6 +1,6 @@ [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C -funcName=generateLongLivedGapTriggered({1000010020, 1000010030, 1000020030, 1000020040, 1010010030}, 5, 10) +funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/hypernuclei.gun", 1) [GeneratorPythia8] config=${O2_ROOT}/share/Generators/egconfig/pythia8_hi.cfg diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun new file mode 100644 index 000000000..d15d11554 --- /dev/null +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun @@ -0,0 +1,8 @@ +# PDG N ptMin ptMax yMin yMax +1000010020 4 0.2 10 -1 1 +1000010030 4 0.2 10 -1 1 +1000020030 4 0.2 10 -1 1 +1000020040 4 0.2 10 -1 1 +1010010030 4 0.2 10 -1 1 +1010010040 4 0.2 10 -1 1 +1010020040 4 0.2 10 -1 1 From 763bc731607f5710a9d31b8dafa7608b3c9517f5 Mon Sep 17 00:00:00 2001 From: fmazzasc Date: Tue, 20 Feb 2024 17:57:38 +0100 Subject: [PATCH 0391/1239] fix --- MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini b/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini index feceea039..4acc1d268 100644 --- a/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini +++ b/MC/config/PWGLF/ini/GeneratorLFHyperNucleiPbPbGap.ini @@ -1,6 +1,6 @@ [GeneratorExternal] fileName=${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator_pythia8_longlived_gaptriggered.C -funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/hypernuclei.gun", 1) +funcName=generateLongLivedGapTriggered("${O2DPG_ROOT}/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun", 1) [GeneratorPythia8] config=${O2_ROOT}/share/Generators/egconfig/pythia8_hi.cfg From c9f8bc00168c7461ec73326ecd7a2cbe5fe72de5 Mon Sep 17 00:00:00 2001 From: fmazzasc Date: Tue, 20 Feb 2024 18:04:24 +0100 Subject: [PATCH 0392/1239] Increase number of injected particles --- .../PWGLF/pythia8/generator/hypernuclei_pbpb.gun | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun index d15d11554..cc68fa63b 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun @@ -1,8 +1,8 @@ # PDG N ptMin ptMax yMin yMax -1000010020 4 0.2 10 -1 1 -1000010030 4 0.2 10 -1 1 -1000020030 4 0.2 10 -1 1 -1000020040 4 0.2 10 -1 1 -1010010030 4 0.2 10 -1 1 -1010010040 4 0.2 10 -1 1 -1010020040 4 0.2 10 -1 1 +1000010020 10 0.2 10 -1 1 +1000010030 10 0.2 10 -1 1 +1000020030 10 0.2 10 -1 1 +1000020040 10 0.2 10 -1 1 +1010010030 10 0.2 10 -1 1 +1010010040 10 0.2 10 -1 1 +1010020040 10 0.2 10 -1 1 From c399748f3c79bc39f06d46604101a1e55914d71a Mon Sep 17 00:00:00 2001 From: fmazzasc Date: Thu, 22 Feb 2024 15:31:02 +0100 Subject: [PATCH 0393/1239] Inject more particles per event --- .../PWGLF/pythia8/generator/hypernuclei_pbpb.gun | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun index cc68fa63b..3b4a8d274 100644 --- a/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun +++ b/MC/config/PWGLF/pythia8/generator/hypernuclei_pbpb.gun @@ -1,8 +1,8 @@ # PDG N ptMin ptMax yMin yMax -1000010020 10 0.2 10 -1 1 -1000010030 10 0.2 10 -1 1 -1000020030 10 0.2 10 -1 1 -1000020040 10 0.2 10 -1 1 -1010010030 10 0.2 10 -1 1 -1010010040 10 0.2 10 -1 1 -1010020040 10 0.2 10 -1 1 +1000010020 20 0.2 10 -1 1 +1000010030 20 0.2 10 -1 1 +1000020030 20 0.2 10 -1 1 +1000020040 20 0.2 10 -1 1 +1010010030 20 0.2 10 -1 1 +1010010040 20 0.2 10 -1 1 +1010020040 20 0.2 10 -1 1 From 045895f64490c42027828618818b5e3f5bbf3981 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 23 Feb 2024 14:41:07 +0100 Subject: [PATCH 0394/1239] Add asyn-label workflows for PRs (#1489) * Add asyn-label workflows for PRs --------- Co-authored-by: Benedikt Volkel --- .github/workflows/async-auto-label.yml | 18 ++++++++++++++++++ .github/workflows/async-list-label.yml | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 .github/workflows/async-auto-label.yml create mode 100644 .github/workflows/async-list-label.yml diff --git a/.github/workflows/async-auto-label.yml b/.github/workflows/async-auto-label.yml new file mode 100644 index 000000000..8ec0659ca --- /dev/null +++ b/.github/workflows/async-auto-label.yml @@ -0,0 +1,18 @@ +--- +name: Apply requested async label + +'on': + issue_comment: + types: + - created + - edited + +permissions: + pull-requests: write # to update labels + +jobs: + apply_async_labels: + name: Apply requested async label + uses: alisw/ali-bot/.github/workflows/async-auto-label.yml@master + permissions: + pull-requests: write # to update labels diff --git a/.github/workflows/async-list-label.yml b/.github/workflows/async-list-label.yml new file mode 100644 index 000000000..97144b6b1 --- /dev/null +++ b/.github/workflows/async-list-label.yml @@ -0,0 +1,20 @@ +--- +name: Collect and print async labels + +'on': + pull_request: + types: + - opened + - reopened + branches: + - master + +permissions: + pull-requests: write # to update labels + +jobs: + list_async_labels: + name: Collect and print async labels + uses: alisw/ali-bot/.github/workflows/async-list-label.yml@master + permissions: + pull-requests: write # to update labels From e8511152b4dd58dbc3098b380d17cc468761d17c Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Fri, 23 Feb 2024 15:40:23 +0100 Subject: [PATCH 0395/1239] Topology generation fixes for staging (#1491) * Only enable CTP lumi scaling if its available * Disable CALIB_TPC_IDC on staging due to lack of resources --- DATA/common/setenv.sh | 2 +- DATA/common/setenv_calib.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index ae7e276f2..61c817a54 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -165,7 +165,7 @@ DISABLE_ROOT_INPUT="--disable-root-input" # Special detector related settings if [[ -z "${TPC_CORR_SCALING:-}" ]]; then # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1> and TPCCorrMap... configurable param TPC_CORR_SCALING= - if [[ $BEAMTYPE == "pp" ]] || [[ $BEAMTYPE == "PbPb" ]]; then TPC_CORR_SCALING+="--lumi-type 1 TPCCorrMap.lumiInstFactor=2.414"; fi + if ( [[ $BEAMTYPE == "pp" ]] || [[ $BEAMTYPE == "PbPb" ]] ) && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1 TPCCorrMap.lumiInstFactor=2.414"; fi if [[ $BEAMTYPE == "cosmic" ]]; then TPC_CORR_SCALING=" TPCCorrMap.lumiMean=-1;"; fi # for COSMICS we disable all corrections export TPC_CORR_SCALING=$TPC_CORR_SCALING fi diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index ae07638c6..95db74f33 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -76,10 +76,10 @@ if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then if [[ $CAN_DO_CALIB_TPC_VDRIFTTGL == 1 ]]; then if [[ -z ${CALIB_TPC_VDRIFTTGL+x} ]]; then CALIB_TPC_VDRIFTTGL=1; fi fi - # IDCs (by default we enable it for running the synch. reco on the EPNs) + # IDCs (by default we enable it for running the synch. reco on the EPNs, but not on staging since we have only 1 calibration node available) if [[ $CAN_DO_CALIB_TPC_IDC == 1 ]]; then if [[ -z ${CALIB_TPC_IDC+x} ]]; then - if [[ $EPNSYNCMODE == 1 ]]; then + if [[ $EPNSYNCMODE == 1 ]] && [[ "${GEN_TOPO_DEPLOYMENT_TYPE:-}" != "ALICE_STAGING" ]]; then CALIB_TPC_IDC=1; else CALIB_TPC_IDC=0; From fbbe5cd3002e47430e632840bba3c60db7067516 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 23 Feb 2024 15:43:46 +0100 Subject: [PATCH 0396/1239] [WF] Take back unneccessary permissions (#1492) Co-authored-by: Benedikt Volkel --- .github/workflows/async-auto-label.yml | 3 +-- .github/workflows/async-list-label.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/async-auto-label.yml b/.github/workflows/async-auto-label.yml index 8ec0659ca..b0a17c7fa 100644 --- a/.github/workflows/async-auto-label.yml +++ b/.github/workflows/async-auto-label.yml @@ -7,8 +7,7 @@ name: Apply requested async label - created - edited -permissions: - pull-requests: write # to update labels +permissions: {} jobs: apply_async_labels: diff --git a/.github/workflows/async-list-label.yml b/.github/workflows/async-list-label.yml index 97144b6b1..6b6f6f57d 100644 --- a/.github/workflows/async-list-label.yml +++ b/.github/workflows/async-list-label.yml @@ -9,8 +9,7 @@ name: Collect and print async labels branches: - master -permissions: - pull-requests: write # to update labels +permissions: {} jobs: list_async_labels: From 77bda58b7e2f0b9988966bb6f74a6d793d3836c6 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 23 Feb 2024 17:37:13 +0100 Subject: [PATCH 0397/1239] [WF] Raise write permission (#1495) --- .github/workflows/async-list-label.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/async-list-label.yml b/.github/workflows/async-list-label.yml index 6b6f6f57d..97144b6b1 100644 --- a/.github/workflows/async-list-label.yml +++ b/.github/workflows/async-list-label.yml @@ -9,7 +9,8 @@ name: Collect and print async labels branches: - master -permissions: {} +permissions: + pull-requests: write # to update labels jobs: list_async_labels: From 1ce5e589d811175cbd5fccfa56e37bc74362d896 Mon Sep 17 00:00:00 2001 From: shahoian Date: Fri, 23 Feb 2024 17:30:20 +0100 Subject: [PATCH 0398/1239] Use alio2-cr1-flp199-ib:8083 as DCS-CCDB server, define in setenv.sh --- DATA/common/setenv.sh | 2 ++ DATA/production/calib/hmp-pedestals-processing.sh | 2 +- DATA/production/calib/its-noise-aggregator.sh | 2 +- DATA/production/calib/its-threshold-aggregator.sh | 2 +- DATA/production/calib/mch-badchannel-aggregator.sh | 2 +- DATA/production/calib/mft-noise-aggregator.sh | 2 +- DATA/production/calib/mid-badchannels.sh | 2 +- DATA/testing/detectors/MID/mid-calib-workflow.sh | 2 +- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/DATA/common/setenv.sh b/DATA/common/setenv.sh index 61c817a54..18307ebf2 100755 --- a/DATA/common/setenv.sh +++ b/DATA/common/setenv.sh @@ -110,6 +110,8 @@ if [[ -z "${IS_TRIGGERED_DATA:-}" ]]; then export IS_TRIGGERED_DATA=0; fi if [[ -z "${CTF_DIR:-}" ]]; then CTF_DIR=$FILEWORKDIR; fi # Directory where to store CTFs if [[ -z "${CALIB_DIR:-}" ]]; then CALIB_DIR="/dev/null"; fi # Directory where to store output from calibration workflows, /dev/null : skip their writing if [[ -z "${EPN2EOS_METAFILES_DIR:-}" ]]; then EPN2EOS_METAFILES_DIR="/dev/null"; fi # Directory where to store epn2eos files metada, /dev/null : skip their writing +if [[ -z "${DCSCCDBSERVER:-}" ]]; then export DCSCCDBSERVER="http://alio2-cr1-flp199-ib:8083"; fi # server for transvering calibration data to DCS + if [[ $EPNSYNCMODE == 0 ]]; then if [[ -z "${SHMSIZE:-}" ]]; then export SHMSIZE=$(( 8 << 30 )); fi # Size of shared memory for messages if [[ -z "${NGPUS:-}" ]]; then export NGPUS=1; fi # Number of GPUs to use, data distributed round-robin diff --git a/DATA/production/calib/hmp-pedestals-processing.sh b/DATA/production/calib/hmp-pedestals-processing.sh index f7c48d737..b14d38660 100755 --- a/DATA/production/calib/hmp-pedestals-processing.sh +++ b/DATA/production/calib/hmp-pedestals-processing.sh @@ -57,7 +57,7 @@ fi SPEC_PARAM="" if [ $HMP_NODCSCCDB_REC == 'false' ]; then - SPEC_PARAM+="--use-dcsccdb --dcsccdb-uri 'http://alio2-cr1-flp199.cern.ch:8083' --dcsccdb-alivehours 3 " + SPEC_PARAM+="--use-dcsccdb --dcsccdb-uri $DCSCCDBSERVER --dcsccdb-alivehours 3 " fi if [ $HMP_CCDB_REC == 'true' ]; then diff --git a/DATA/production/calib/its-noise-aggregator.sh b/DATA/production/calib/its-noise-aggregator.sh index 97bfcf696..e6aeb04d9 100755 --- a/DATA/production/calib/its-noise-aggregator.sh +++ b/DATA/production/calib/its-noise-aggregator.sh @@ -17,7 +17,7 @@ fi if [[ -z $NTHREADS ]] ; then NTHREADS=1; fi CCDBPATH1="http://o2-ccdb.internal" -CCDBPATH2="http://alio2-cr1-flp199.cern.ch:8083" +CCDBPATH2="$DCSCCDBSERVER" if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" || ! -z $ISTEST ]]; then CCDBPATH1="http://ccdb-test.cern.ch:8080" CCDBPATH2="http://ccdb-test.cern.ch:8080" diff --git a/DATA/production/calib/its-threshold-aggregator.sh b/DATA/production/calib/its-threshold-aggregator.sh index cc788d79e..81e1b11c6 100755 --- a/DATA/production/calib/its-threshold-aggregator.sh +++ b/DATA/production/calib/its-threshold-aggregator.sh @@ -11,7 +11,7 @@ PROXY_INSPEC="tunestring:ITS/TSTR;runtype:ITS/RUNT;fittype:ITS/FITT;scantype:ITS CCDBPATH1="" CCDBPATH2="" if [ $RUNTYPE_ITS == "tuning" ] || [ $RUNTYPE_ITS == "digital" ] || [ $RUNTYPE_ITS == "tuningbb" ]; then - CCDBPATH1="http://alio2-cr1-flp199.cern.ch:8083" + CCDBPATH1="$DCSCCDBSERVER" CCDBPATH2="http://o2-ccdb.internal" else CCDBPATH1="http://o2-ccdb.internal" diff --git a/DATA/production/calib/mch-badchannel-aggregator.sh b/DATA/production/calib/mch-badchannel-aggregator.sh index e362e373a..8af400b9d 100755 --- a/DATA/production/calib/mch-badchannel-aggregator.sh +++ b/DATA/production/calib/mch-badchannel-aggregator.sh @@ -20,7 +20,7 @@ BADCHANNEL_CONFIG="${ARGS_ALL_CONFIG};MCHBadChannelCalibratorParam.maxPed=${MCH_ WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name mch-badchannel-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=mch-badchannel-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " WORKFLOW+="o2-calibration-mch-badchannel-calib-workflow $ARGS_ALL --configKeyValues \"$BADCHANNEL_CONFIG\" | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ali-calib-dcs.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " +WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$DCSCCDBSERVER\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " add_QC_from_consul "/o2/components/qc/ANY/any/mch-badchannel" "" WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" diff --git a/DATA/production/calib/mft-noise-aggregator.sh b/DATA/production/calib/mft-noise-aggregator.sh index 3fa908025..8c2ef1290 100755 --- a/DATA/production/calib/mft-noise-aggregator.sh +++ b/DATA/production/calib/mft-noise-aggregator.sh @@ -11,7 +11,7 @@ PROXY_INSPEC="A:MFT/DIGITS/0;B:MFT/DIGITSROF/0" WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --proxy-name mft-noise-input-proxy --dataspec \"$PROXY_INSPEC\" --network-interface ib0 --channel-config \"name=mft-noise-input-proxy,method=bind,type=pull,rateLogging=0,transport=zeromq\" | " WORKFLOW+="o2-calibration-mft-calib-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --useDigits --prob-threshold 1e-5 --send-to-server DCS-CCDB --path-CCDB \"/MFT/Calib/NoiseMap\" --path-DCS \"/MFT/Config/NoiseMap\" | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://ali-calib-dcs.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " +WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"$DCSCCDBSERVER\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" if [ $WORKFLOWMODE == "print" ]; then diff --git a/DATA/production/calib/mid-badchannels.sh b/DATA/production/calib/mid-badchannels.sh index 6060b4116..edc18eaec 100755 --- a/DATA/production/calib/mid-badchannels.sh +++ b/DATA/production/calib/mid-badchannels.sh @@ -26,7 +26,7 @@ if [[ -z ${CCDB_POPULATOR_UPLOAD_PATH} ]]; then CCDB_POPULATOR_UPLOAD_PATH_DCS="$CCDB_POPULATOR_UPLOAD_PATH" else CCDB_POPULATOR_UPLOAD_PATH="http://o2-ccdb.internal" - CCDB_POPULATOR_UPLOAD_PATH_DCS="http://alio2-cr1-flp199.cern.ch:8083" + CCDB_POPULATOR_UPLOAD_PATH_DCS="$DCSCCDBSERVER" fi fi if [[ "${GEN_TOPO_VERBOSE:-}" == "1" ]]; then diff --git a/DATA/testing/detectors/MID/mid-calib-workflow.sh b/DATA/testing/detectors/MID/mid-calib-workflow.sh index e846ea94f..a9ad5a7a7 100755 --- a/DATA/testing/detectors/MID/mid-calib-workflow.sh +++ b/DATA/testing/detectors/MID/mid-calib-workflow.sh @@ -7,7 +7,7 @@ WORKFLOW="o2-dpl-raw-proxy $ARGS_ALL --dataspec \"$MID_RAW_PROXY_INSPEC\" --inje WORKFLOW+="o2-mid-raw-to-digits-workflow $ARGS_ALL $MID_RAW_TO_DIGITS_OPTS | " WORKFLOW+="o2-mid-calibration-workflow $ARGS_ALL | " WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://o2-ccdb.internal\" --sspec-min 0 --sspec-max 0 | " -WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"http://alio2-cr1-flp199.cern.ch:8083\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " +WORKFLOW+="o2-calibration-ccdb-populator-workflow $ARGS_ALL --configKeyValues \"$ARGS_ALL_CONFIG\" --ccdb-path=\"${DCSCCDBSERVER:-http://alio2-cr1-flp199-ib:8083}\" --sspec-min 1 --sspec-max 1 --name-extention dcs | " WORKFLOW+="o2-dpl-run $ARGS_ALL $GLOBALDPLOPT" if [ "$WORKFLOWMODE" == "print" ]; then From 340b4fe7720e0b87b377d1a16910f4adf2f5daa8 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 23 Feb 2024 17:56:21 +0100 Subject: [PATCH 0399/1239] [WF] Run on pull_request_target (#1496) Co-authored-by: Benedikt Volkel --- .github/workflows/async-list-label.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/async-list-label.yml b/.github/workflows/async-list-label.yml index 97144b6b1..b53ec6e6e 100644 --- a/.github/workflows/async-list-label.yml +++ b/.github/workflows/async-list-label.yml @@ -2,15 +2,14 @@ name: Collect and print async labels 'on': - pull_request: + pull_request_target: types: - opened - reopened branches: - master -permissions: - pull-requests: write # to update labels +permissions: {} jobs: list_async_labels: From a09b91ff6db85564d63d3f186b0aebdf01956978 Mon Sep 17 00:00:00 2001 From: Francesco Mazzaschi <43742195+fmazzasc@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:44:03 +0100 Subject: [PATCH 0400/1239] Enable strangeness tracking in strange particle simulations (#1499) --- MC/run/PWGLF/run_HyperNucleiInjectedGap.sh | 2 +- MC/run/PWGLF/run_StrangenessInjected.sh | 2 +- MC/run/PWGLF/run_StrangenessTriggered.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh b/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh index cbf094547..04659f024 100644 --- a/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh +++ b/MC/run/PWGLF/run_HyperNucleiInjectedGap.sh @@ -26,7 +26,7 @@ ENERGY=${ENERGY:-13600} # create workflow ${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM ${ENERGY} -col ${SYSTEM} -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} -confKey "Diamond.width[0]=0.1;Diamond.width[1]=0.1;Diamond.width[2]=6." -e ${SIMENGINE} ${SEED} -mod "--skipModules ZDC" \ - -ini ${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFHyperNuclei${SYSTEM}Gap.ini + -ini ${O2DPG_ROOT}/MC/config/PWGLF/ini/GeneratorLFHyperNuclei${SYSTEM}Gap.ini --with-strangeness-tracking # run workflow # allow increased timeframe parallelism with --cpu-limit 32 diff --git a/MC/run/PWGLF/run_StrangenessInjected.sh b/MC/run/PWGLF/run_StrangenessInjected.sh index 9c07c9271..e8cb17068 100755 --- a/MC/run/PWGLF/run_StrangenessInjected.sh +++ b/MC/run/PWGLF/run_StrangenessInjected.sh @@ -35,7 +35,7 @@ O2_SIM_WORKFLOW=${O2_SIM_WORKFLOW:-"${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py"} $O2_SIM_WORKFLOW -eCM ${ENERGY} -col ${SYSTEM} -gen external \ -j ${NWORKERS} \ -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} \ - -confKey "Diamond.width[2]=6." \ + -confKey "Diamond.width[2]=6." --with-strangeness-tracking \ ${SEED} \ -procBkg "inel" -colBkg $SYSTEM --embedding -nb ${NBKGEVENTS} -genBkg pythia8 \ -e ${SIMENGINE} \ diff --git a/MC/run/PWGLF/run_StrangenessTriggered.sh b/MC/run/PWGLF/run_StrangenessTriggered.sh index 412edf995..39ff2ff1e 100755 --- a/MC/run/PWGLF/run_StrangenessTriggered.sh +++ b/MC/run/PWGLF/run_StrangenessTriggered.sh @@ -34,7 +34,7 @@ O2_SIM_WORKFLOW=${O2_SIM_WORKFLOW:-"${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py"} $O2_SIM_WORKFLOW -eCM ${ENERGY} -col ${SYSTEM} -gen external \ -j ${NWORKERS} \ -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -interactionRate ${INTRATE} \ - -confKey "Diamond.width[2]=6." \ + -confKey "Diamond.width[2]=6." --with-strangeness-tracking \ ${SEED} \ -e ${SIMENGINE} \ -ini $CFGINIFILE From f568fd496d33b39d46bc6f6582f56269f946f2bd Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 27 Feb 2024 14:09:19 +0100 Subject: [PATCH 0401/1239] [SimCI] Make mem limit configurable in WF tests (#1502) --- test/run_workflow_tests.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index e96b07141..15746b3ce 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -74,10 +74,11 @@ test_single_wf() if [[ "${ret_this}" != "0" ]] ; then echo "[FATAL]: O2DPG_TEST Workflow creation failed" >> ${LOG_FILE_WF} elif [[ "${execute}" != "" ]] ; then - ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 -tt aod >> ${LOG_FILE_WF} 2>&1 + local memlimit=${O2DPG_TEST_WORKFLOW_MEMLIMIT:+--mem-limit O2DPG_TEST_WORKFLOW_MEMLIMIT} + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 -tt aod ${memlimit} >> ${LOG_FILE_WF} 2>&1 ret_this=${?} - [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } - [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; } + [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } + [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; } ret_this=$((ret_this + ret_this_qc + ret_this_analysis)) [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} fi @@ -173,6 +174,8 @@ print_usage() echo " If also not set, this will be set to HEAD. However, if there are unstaged" echo " changes, it will left blank." echo + echo " O2DPG_TEST_WORKFLOW_MEMLIMIT : The memory limit that is passed to the workflow runner in case a workflow is executed (optional)" + echo } while [ "$1" != "" ] ; do From 8c1886416bdef6ea80d0f0cbe976c6211e34e72a Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 27 Feb 2024 15:47:12 +0100 Subject: [PATCH 0402/1239] [SimCI] Make a variable (#1503) --- test/run_workflow_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index 15746b3ce..2acde102e 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -74,7 +74,7 @@ test_single_wf() if [[ "${ret_this}" != "0" ]] ; then echo "[FATAL]: O2DPG_TEST Workflow creation failed" >> ${LOG_FILE_WF} elif [[ "${execute}" != "" ]] ; then - local memlimit=${O2DPG_TEST_WORKFLOW_MEMLIMIT:+--mem-limit O2DPG_TEST_WORKFLOW_MEMLIMIT} + local memlimit=${O2DPG_TEST_WORKFLOW_MEMLIMIT:+--mem-limit ${O2DPG_TEST_WORKFLOW_MEMLIMIT}} ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 -tt aod ${memlimit} >> ${LOG_FILE_WF} 2>&1 ret_this=${?} [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } From dfc147c8f73c02a975003baa196b00fbdca153b1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 15 Feb 2024 18:39:39 +0100 Subject: [PATCH 0403/1239] Revert "Temporary workaround, to be reverted once O2 PR 12412 is in all async tags" This reverts commit fd619fe62c5c4f622492992a367da6945f1d5186. --- DATA/production/configurations/asyncReco/async_pass.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 9d1f49025..a49a941b0 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -371,8 +371,6 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ $ALIEN_JDL_UNOPTIMIZEDGPUSETTINGS != 1 ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_1gpu # sets the multiplicities to optimized defaults for this configuration (1 job with 1 gpu on EPNs) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 - export TIMEFRAME_RATE_LIMIT=8 # WORKAROUND: Needed until O2 PR 12412 is in all async tags - export SHMSIZE=30000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags else # forcing multiplicities to be 1 export MULTIPLICITY_PROCESS_tof_matcher=1 @@ -392,13 +390,9 @@ if [[ -n "$ALIEN_JDL_USEGPUS" && $ALIEN_JDL_USEGPUS != 0 ]] ; then if [[ $BEAMTYPE == "pp" ]]; then export OPTIMIZED_PARALLEL_ASYNC=pp_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, pp) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 - export TIMEFRAME_RATE_LIMIT=45 # WORKAROUND: Needed until O2 PR 12412 is in all async tags - export SHMSIZE=100000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags else # PbPb export OPTIMIZED_PARALLEL_ASYNC=PbPb_4gpu # sets the multiplicities to optimized defaults for this configuration (1 Numa, PbPb) export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 - export TIMEFRAME_RATE_LIMIT=30 # WORKAROUND: Needed until O2 PR 12412 is in all async tags - export SHMSIZE=100000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags fi fi fi @@ -428,8 +422,6 @@ else else export OPTIMIZED_PARALLEL_ASYNC=pp_64cpu # to use EPNs with full NUMA domain but without GPUs export OPTIMIZED_PARALLEL_ASYNC_AUTO_SHM_LIMIT=1 - export TIMEFRAME_RATE_LIMIT=32 # WORKAROUND: Needed until O2 PR 12412 is in all async tags - export SHMSIZE=90000000000 # WORKAROUND: Needed until O2 PR 12412 is in all async tags fi fi fi From ffe6861adc210e9828278702dd5b372342224685 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 26 Feb 2024 14:56:46 +0100 Subject: [PATCH 0404/1239] Do not ask for geometry in GLO ITS-TPC and TOF matching QC --- DATA/production/qc-async/itstpc.json | 2 +- DATA/production/qc-async/itstpctof.json | 2 +- DATA/production/qc-async/itstpctofwtrd.json | 4 ++-- DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json | 2 +- DATA/production/qc-sync/itstpctof.json | 2 +- DATA/production/qc-sync/itstpctrdtof.json | 2 +- MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json | 4 ++-- MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json | 2 +- .../QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/DATA/production/qc-async/itstpc.json b/DATA/production/qc-async/itstpc.json index b294b8cf9..59607bc4d 100644 --- a/DATA/production/qc-async/itstpc.json +++ b/DATA/production/qc-async/itstpc.json @@ -58,7 +58,7 @@ "etaCut": "1e10f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/DATA/production/qc-async/itstpctof.json b/DATA/production/qc-async/itstpctof.json index a843da3ea..c5c8697af 100644 --- a/DATA/production/qc-async/itstpctof.json +++ b/DATA/production/qc-async/itstpctof.json @@ -46,7 +46,7 @@ "minDCACutY" : "10.f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/DATA/production/qc-async/itstpctofwtrd.json b/DATA/production/qc-async/itstpctofwtrd.json index 55094c2ad..2c397df57 100644 --- a/DATA/production/qc-async/itstpctofwtrd.json +++ b/DATA/production/qc-async/itstpctofwtrd.json @@ -46,7 +46,7 @@ "minDCACutY" : "10.f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", @@ -55,7 +55,7 @@ "askOnceAllButField": "true", "needPropagatorD": "false" }, - "saveObjectsToFile" : "TOFmatchedITSTPCTOF_TPCTOF.root", + "saveObjectsToFile" : "TOFmatchedITSTPCTOF_TPCTOF_wTRD.root", "" : "For debugging, path to the file where to save. If empty or missing it won't save." } } diff --git a/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json b/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json index da8964a41..53a1dca39 100644 --- a/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json +++ b/DATA/production/qc-sync/glo-itstpc-mtch-qcmn-epn.json @@ -44,7 +44,7 @@ "minDCACutY" : "10.f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/DATA/production/qc-sync/itstpctof.json b/DATA/production/qc-sync/itstpctof.json index dcc986436..fefe85c87 100644 --- a/DATA/production/qc-sync/itstpctof.json +++ b/DATA/production/qc-sync/itstpctof.json @@ -53,7 +53,7 @@ "minDCACutY" : "10.f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/DATA/production/qc-sync/itstpctrdtof.json b/DATA/production/qc-sync/itstpctrdtof.json index 2fb6363ba..087449715 100644 --- a/DATA/production/qc-sync/itstpctrdtof.json +++ b/DATA/production/qc-sync/itstpctrdtof.json @@ -53,7 +53,7 @@ "minDCACutY" : "10.f" }, "grpGeomRequest" : { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json index 27dff87b8..5aba15f8a 100644 --- a/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json +++ b/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json @@ -56,7 +56,7 @@ "isMC": "true" }, "grpGeomRequest": { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", @@ -72,4 +72,4 @@ } }, "dataSamplingPolicies": [] -} \ No newline at end of file +} diff --git a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json index 4d5acc87a..7ae8f16e6 100644 --- a/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json @@ -49,7 +49,7 @@ "minDCACutY": "10.f" }, "grpGeomRequest": { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", diff --git a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json index ef0d21311..95f6f9008 100644 --- a/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json +++ b/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json @@ -50,7 +50,7 @@ "minDCACutY": "10.f" }, "grpGeomRequest": { - "geomRequest": "Aligned", + "geomRequest": "None", "askGRPECS": "false", "askGRPLHCIF": "false", "askGRPMagField": "true", From 95c2b213b346569b6c93f7cde096cfd8dd5ea6ff Mon Sep 17 00:00:00 2001 From: shahoian Date: Wed, 28 Feb 2024 17:48:27 +0100 Subject: [PATCH 0405/1239] Move RECO_NUM_NODES_WORKFLOW_CMP definition to the beginning --- DATA/production/workflow-multiplicities.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DATA/production/workflow-multiplicities.sh b/DATA/production/workflow-multiplicities.sh index 275200d4f..dd04d93ba 100644 --- a/DATA/production/workflow-multiplicities.sh +++ b/DATA/production/workflow-multiplicities.sh @@ -30,6 +30,8 @@ if [[ $SYNCMODE == 1 ]]; then NTRDTRKTHREADS=1; else NTRDTRKTHREADS=; fi : ${NGPURECOTHREADS:=-1} # -1 = auto-detect +RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? ($RECO_NUM_NODES_WORKFLOW < 230 ? $RECO_NUM_NODES_WORKFLOW : 230) : 15) * ($NUMAGPUIDS != 0 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains + # --------------------------------------------------------------------------------------------------------------------- # Process multiplicities @@ -157,7 +159,6 @@ elif [[ $EPNPIPELINES != 0 ]]; then NTRDTRKTHREADS=2 ITSTRK_THREADS=2 ITSTPC_THREADS=2 - RECO_NUM_NODES_WORKFLOW_CMP=$((($RECO_NUM_NODES_WORKFLOW > 15 ? ($RECO_NUM_NODES_WORKFLOW < 230 ? $RECO_NUM_NODES_WORKFLOW : 230) : 15) * ($NUMAGPUIDS != 0 ? 2 : 1))) # Limit the lower scaling factor, multiply by 2 if we have 2 NUMA domains # Tuned multiplicities for sync pp / Pb-Pb processing if [[ $BEAMTYPE == "pp" ]]; then N_ITSRAWDEC=$(math_max $((6 * $EPNPIPELINES * $NGPUS / 4)) 1) From d47944257ead124fdd7b33b1430c8a8e5956da59 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 28 Feb 2024 23:48:07 +0100 Subject: [PATCH 0406/1239] To allow to set the trackQC sampling from the JDL, or have a subsample with full sampling --- .../configurations/asyncReco/async_pass.sh | 2 +- .../configurations/asyncReco/setenv_extra.sh | 24 ++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index a49a941b0..ce6d3ce5e 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -20,7 +20,7 @@ run_AOD_merging() { timeStartFullProcessing=`date +%s` # to skip positional arg parsing before the randomizing part. -inputarg="${1}" +export inputarg="${1}" if [[ "${1##*.}" == "root" ]]; then #echo ${1##*.} diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index 4f9992a67..c1d82bbca 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -622,7 +622,29 @@ fi # ad-hoc settings for AOD echo ALIEN_JDL_LPMPRODUCTIONTAG = $ALIEN_JDL_LPMPRODUCTIONTAG echo ALIEN_JDL_LPMPASSNAME = $ALIEN_JDL_LPMPASSNAME -export ARGS_EXTRA_PROCESS_o2_aod_producer_workflow="$ARGS_EXTRA_PROCESS_o2_aod_producer_workflow --aod-writer-maxfilesize $AOD_FILE_SIZE --lpmp-prod-tag $ALIEN_JDL_LPMPRODUCTIONTAG --reco-pass $ALIEN_JDL_LPMPASSNAME" +# Track QC table sampling +if [[ -n $ALIEN_JDL_TRACKQCFRACTION ]]; then + TRACKQC_FRACTION=$ALIEN_JDL_TRACKQCFRACTION +else + if [[ $ALIEN_JDL_ENABLEPERMILFULLTRACKQC == "1" ]]; then + PERMIL_FULLTRACKQC=${ALIEN_JDL_PERMILFULLTRACKQC:-100} + INVERSE_PERMIL_FULLTRACKQC=$((1000/PERMIL_FULLTRACKQC)) + if [[ -f wn.xml ]]; then + HASHCODE=`grep alien:// wn.xml | tr ' ' '\n' | grep ^lfn | cut -d\" -f2 | head -1 | cksum | cut -d ' ' -f 1` + else + HASHCODE=`echo "${inputarg}" | cksum | cut -d ' ' -f 1` + fi + if [[ "$((HASHCODE%INVERSE_PERMIL_FULLTRACKQC))" -eq "0" ]]; then + TRACKQC_FRACTION=1 + else + TRACKQC_FRACTION=0.1 + fi + else + TRACKQC_FRACTION=0.1 + fi +fi +echo TRACKQC_FRACTION = $TRACKQC_FRACTION +export ARGS_EXTRA_PROCESS_o2_aod_producer_workflow="$ARGS_EXTRA_PROCESS_o2_aod_producer_workflow --aod-writer-maxfilesize $AOD_FILE_SIZE --lpmp-prod-tag $ALIEN_JDL_LPMPRODUCTIONTAG --reco-pass $ALIEN_JDL_LPMPASSNAME --trackqc-fraction $TRACKQC_FRACTION" if [[ $PERIOD == "LHC22c" ]] || [[ $PERIOD == "LHC22d" ]] || [[ $PERIOD == "LHC22e" ]] || [[ $PERIOD == "JUN" ]] || [[ $PERIOD == "LHC22f" ]] || [[ $PERIOD == "LHC22m" ]] || [[ "$RUNNUMBER" == @(526463|526465|526466|526467|526468|526486|526505|526508|526510|526512|526525|526526|526528|526534|526559|526596|526606|526612|526638|526639|526641|526643|526647|526649|526689|526712|526713|526714|526715|526716|526719|526720|526776|526886|526926|526927|526928|526929|526934|526935|526937|526938|526963|526964|526966|526967|526968|527015|527016|527028|527031|527033|527034|527038|527039|527041|527057|527076|527108|527109|527228|527237|527259|527260|527261|527262|527345|527347|527349|527446|527518|527523|527734) ]] ; then export ARGS_EXTRA_PROCESS_o2_aod_producer_workflow="$ARGS_EXTRA_PROCESS_o2_aod_producer_workflow --ctpreadout-create 1" fi From 2b9b08709777b7f85bdd1282eb8c06519c361c34 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Thu, 29 Feb 2024 09:47:08 +0100 Subject: [PATCH 0407/1239] Do not split the metrics, as it is too slow, unless requested --- .../configurations/asyncReco/async_pass.sh | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index ce6d3ce5e..bd6e9ce89 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -610,21 +610,23 @@ else fi # now extract all performance metrics -IFS=$'\n' -timeStart=`date +%s` -for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json ; do - suffix=`echo $perfMetricsFiles | sed 's/performanceMetrics\(.*\).json/\1/'` - if [[ -f "performanceMetrics.json" ]]; then - for workflow in `grep ': {' $perfMetricsFiles`; do - strippedWorkflow=`echo $workflow | cut -d\" -f2` - cat $perfMetricsFiles | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics${suffix}.json - done - fi -done -timeEnd=`date +%s` -timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) -delta=$(( $timeEnd-$timeStart )) -echo "Time spent in splitting the metrics files = $delta s" +if [[ $ALIEN_JDL_EXTRACTMETRICS == "1" ]]; then + IFS=$'\n' + timeStart=`date +%s` + for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json ; do + suffix=`echo $perfMetricsFiles | sed 's/performanceMetrics\(.*\).json/\1/'` + if [[ -f "performanceMetrics.json" ]]; then + for workflow in `grep ': {' $perfMetricsFiles`; do + strippedWorkflow=`echo $workflow | cut -d\" -f2` + cat $perfMetricsFiles | jq '.'\"${strippedWorkflow}\"'' > ${strippedWorkflow}_metrics${suffix}.json + done + fi + done + timeEnd=`date +%s` + timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) + delta=$(( $timeEnd-$timeStart )) + echo "Time spent in splitting the metrics files = $delta s" +fi if [[ $ALIEN_JDL_AODOFF != 1 ]]; then # flag to possibly enable Analysis QC From b04143af8dbefc4befc2853eba74a2ce63e21846 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 28 Feb 2024 22:20:45 +0100 Subject: [PATCH 0408/1239] Running extraction of TPC time series in anchored MC --- MC/run/ANCHOR/anchorMC.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 2bab586de..c30597eb6 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -235,6 +235,11 @@ echo "Ready to start main workflow" ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} MCRC=$? # <--- we'll report back this code +if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then + echo "Running TPC time series" + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt tpctimes +fi + [[ ! -z "${DISABLE_QC}" ]] && echo "INFO: QC is disabled, skip it." if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then From b9cb67a0845587b3d34bb6d320711dc0478c4022 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Thu, 29 Feb 2024 17:49:32 +0100 Subject: [PATCH 0409/1239] [Anchor] Add some more help messages (#1511) * help messages * set ALIEN_JDL_ADDTIMESERIESINMC to a value at the beginning --- MC/run/ANCHOR/anchorMC.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index c30597eb6..13512fea7 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -41,10 +41,13 @@ print_help() echo "PRODSPLIT." echo echo "Optional are:" - echo "NWORKERS," - echo "ALIEN_JDL_CPULIMIT or CPULIMIT," - echo "ALIEN_JDL_SIMENGINE or SIMENGINE." - echo "DISABLE_QC (set this to disable QC, e.g. DISABLE_QC=1)" + echo "ALIEN_JDL_CPULIMIT or CPULIMIT, set the CPU limit of the workflow runner, default: 8," + echo "NWORKERS, set the number of workers during detector transport, default: 8," + echo "ALIEN_JDL_SIMENGINE or SIMENGINE, choose the transport engine, default: TGeant4," + echo "ALIEN_JDL_WORKFLOWDETECTORS, set detectors to be taken into account, default: ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP," + echo "ALIEN_JDL_ANCHOR_SIM_OPTIONS, additional options that are passed to the workflow creation, default: -gen pythia8," + echo "ALIEN_JDL_ADDTIMESERIESINMC, run TPC time series. Default: 1, switch off by setting to 0," + echo "DISABLE_QC, set this to disable QC, e.g. to 1" } # Prevent the script from being soured to omit unexpected surprises when exit is used @@ -93,6 +96,8 @@ export ALIEN_JDL_LPMPRODUCTIONTAG=${ALIEN_JDL_LPMPRODUCTIONTAG:-${PRODUCTIONTAG} export ALIEN_JDL_LPMANCHORRUN=${ALIEN_JDL_LPMANCHORRUN:-${ANCHORRUN}} export ALIEN_JDL_LPMANCHORPRODUCTION=${ALIEN_JDL_LPMANCHORPRODUCTION:-${ANCHORPRODUCTION}} export ALIEN_JDL_LPMANCHORYEAR=${ALIEN_JDL_LPMANCHORYEAR:-${ANCHORYEAR}} +# decide whether to run TPC time series; on by default, switched off by setting to 0 +export ALIEN_JDL_ADDTIMESERIESINMC=${ALIEN_JDL_ADDTIMESERIESINMC:-1} # cache the production tag, will be set to a special anchor tag; reset later in fact ALIEN_JDL_LPMPRODUCTIONTAG_KEEP=$ALIEN_JDL_LPMPRODUCTIONTAG @@ -236,6 +241,7 @@ ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_ MCRC=$? # <--- we'll report back this code if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then + # Default value is 1 so this is run by default. echo "Running TPC time series" ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt tpctimes fi From b818d3447fc3f65c4b4cf8f33088b9232862887c Mon Sep 17 00:00:00 2001 From: shahoian Date: Thu, 29 Feb 2024 23:39:42 +0100 Subject: [PATCH 0410/1239] Add ITS/MFT time-dependent DeadMaps calibration --- DATA/common/setenv_calib.sh | 22 ++++++++++++++++++- .../configurations/asyncReco/setenv_extra.sh | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/DATA/common/setenv_calib.sh b/DATA/common/setenv_calib.sh index 95db74f33..a2da3ec46 100755 --- a/DATA/common/setenv_calib.sh +++ b/DATA/common/setenv_calib.sh @@ -13,6 +13,8 @@ SOURCE_GUARD_SETENV_CALIB=1 # define the conditions for each calibration if has_detector_calib ITS && has_detectors_reco ITS && has_detector_matching PRIMVTX && [[ ! -z "$VERTEXING_SOURCES" ]]; then CAN_DO_CALIB_PRIMVTX_MEANVTX=1; else CAN_DO_CALIB_PRIMVTX_MEANVTX=0; fi +if has_detector_calib ITS ; then CAN_DO_CALIB_ITS_DEADMAP_TIME=1; else CAN_DO_CALIB_ITS_DEADMAP_TIME=0; fi +if has_detector_calib MFT ; then CAN_DO_CALIB_MFT_DEADMAP_TIME=1; else CAN_DO_CALIB_MFT_DEADMAP_TIME=0; fi if has_detector_calib TOF && has_detector_reco TOF; then CAN_DO_CALIB_TOF_DIAGNOSTICS=1; CAN_DO_CALIB_TOF_INTEGRATEDCURR=1; else CAN_DO_CALIB_TOF_DIAGNOSTICS=0; CAN_DO_CALIB_TOF_INTEGRATEDCURR=0; fi if has_detector_calib TOF && has_detector_reco TOF && ( ( has_detectors_reco ITS TPC && has_detector_matching ITSTPCTOF ) || ( has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPCTRDTOF ) ); then CAN_DO_CALIB_TOF_LHCPHASE=1; CAN_DO_CALIB_TOF_CHANNELOFFSETS=1; else CAN_DO_CALIB_TOF_LHCPHASE=0; CAN_DO_CALIB_TOF_CHANNELOFFSETS=0; fi if has_detector_calib TPC && has_detectors ITS TPC TOF TRD && has_detector_matching ITSTPCTRDTOF; then CAN_DO_CALIB_TPC_SCDCALIB=1; else CAN_DO_CALIB_TPC_SCDCALIB=0; fi @@ -48,7 +50,17 @@ if [[ $BEAMTYPE != "cosmic" ]] || [[ ${FORCECALIBRATIONS:-} == 1 ]] ; then if [[ $CAN_DO_CALIB_PRIMVTX_MEANVTX == 1 ]]; then if [[ -z ${CALIB_PRIMVTX_MEANVTX+x} ]]; then CALIB_PRIMVTX_MEANVTX=1; fi fi - + + # calibrations for ITS + if [[ $CAN_DO_CALIB_ITS_DEADMAP_TIME == 1 ]]; then + if [[ -z ${CALIB_ITS_DEADMAP_TIME+x} ]]; then CALIB_ITS_DEADMAP_TIME=1; fi + fi + + # calibrations for MFT + if [[ $CAN_DO_CALIB_MFT_DEADMAP_TIME == 1 ]]; then + if [[ -z ${CALIB_MFT_DEADMAP_TIME+x} ]]; then CALIB_MFT_DEADMAP_TIME=1; fi + fi + # calibrations for TOF if [[ $CAN_DO_CALIB_TOF_DIAGNOSTICS == 1 ]]; then if [[ -z ${CALIB_TOF_DIAGNOSTICS+x} ]]; then CALIB_TOF_DIAGNOSTICS=1; fi @@ -185,6 +197,8 @@ fi ( [[ -z ${CALIB_PHS_L1PHASE:-} ]] || [[ $CAN_DO_CALIB_PHS_L1PHASE == 0 ]] ) && CALIB_PHS_L1PHASE=0 ( [[ -z ${CALIB_CPV_GAIN:-} ]] || [[ $CAN_DO_CALIB_CPV_GAIN == 0 ]] ) && CALIB_CPV_GAIN=0 ( [[ -z ${CALIB_ZDC_TDC:-} ]] || [[ $CAN_DO_CALIB_ZDC_TDC == 0 ]] ) && CALIB_ZDC_TDC=0 +( [[ -z ${CALIB_ITS_DEADMAP_TIME:-} ]] || [[ $CAN_DO_CALIB_ITS_DEADMAP_TIME == 0 ]] ) && CALIB_ITS_DEADMAP_TIME=0 +( [[ -z ${CALIB_MFT_DEADMAP_TIME:-} ]] || [[ $CAN_DO_CALIB_MFT_DEADMAP_TIME == 0 ]] ) && CALIB_MFT_DEADMAP_TIME=0 # for async: ( [[ -z ${CALIB_EMC_ASYNC_RECALIB:-} ]] || [[ $CAN_DO_CALIB_EMC_ASYNC_RECALIB == 0 ]] ) && CALIB_EMC_ASYNC_RECALIB=0 ( [[ -z ${CALIB_ASYNC_EXTRACTTPCCURRENTS:-} ]] || [[ $CAN_DO_CALIB_ASYNC_EXTRACTTPCCURRENTS == 0 ]] ) && CALIB_ASYNC_EXTRACTTPCCURRENTS=0 @@ -228,6 +242,12 @@ if [[ -z ${CALIBDATASPEC_BARREL_TF:-} ]]; then # prim vtx if [[ $CALIB_PRIMVTX_MEANVTX == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "pvtx:GLO/PVTX/0"; fi + # ITS + if [[ $CALIB_ITS_DEADMAP_TIME == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "itsChipStatus:ITS/CHIPSSTATUS/0"; fi + + # MFT + if [[ $CALIB_MFT_DEADMAP_TIME == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "mftChipStatus:MFT/CHIPSSTATUS/0"; fi + # TOF if [[ $CALIB_TOF_LHCPHASE == 1 ]] || [[ $CALIB_TOF_CHANNELOFFSETS == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "calibTOF:TOF/CALIBDATA/0"; fi if [[ $CALIB_TOF_DIAGNOSTICS == 1 ]]; then add_semicolon_separated CALIBDATASPEC_BARREL_TF "diagWords:TOF/DIAFREQ/0"; fi diff --git a/DATA/production/configurations/asyncReco/setenv_extra.sh b/DATA/production/configurations/asyncReco/setenv_extra.sh index c1d82bbca..3a26bc4d8 100644 --- a/DATA/production/configurations/asyncReco/setenv_extra.sh +++ b/DATA/production/configurations/asyncReco/setenv_extra.sh @@ -557,6 +557,8 @@ if [[ $ADD_CALIB == "1" ]]; then export CALIB_FV0_INTEGRATEDCURR=0 export CALIB_FDD_INTEGRATEDCURR=0 export CALIB_TOF_INTEGRATEDCURR=0 + export CALIB_ITS_DEADMAP_TIME=0 + export CALIB_MFT_DEADMAP_TIME=0 if [[ $DO_TPC_RESIDUAL_EXTRACTION == "1" ]]; then export CALIB_TPC_SCDCALIB=1 export CALIB_TPC_SCDCALIB_SENDTRKDATA=1 From 5510ade973ccd2092cc18554ea075f902b21129e Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 1 Mar 2024 12:20:05 +0100 Subject: [PATCH 0411/1239] [AnalysisQC] Make common args better adjustable (#1514) Co-authored-by: Benedikt Volkel --- MC/analysis_testing/o2dpg_analysis_test_utils.py | 9 ++++++++- MC/analysis_testing/o2dpg_analysis_test_workflow.py | 2 +- MC/config/analysis_testing/json/analyses_config.json | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/MC/analysis_testing/o2dpg_analysis_test_utils.py b/MC/analysis_testing/o2dpg_analysis_test_utils.py index 1a3901fe6..ee896f12f 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_utils.py +++ b/MC/analysis_testing/o2dpg_analysis_test_utils.py @@ -69,7 +69,7 @@ def full_ana_name(raw_ana_name): return f"{ANALYSIS_LABEL}_{raw_ana_name}" -def get_common_args_as_string(analysis_name, all_common_args): +def get_common_args_as_string(ana, all_common_args): """ all_common_args is of the form [-shm-segment-size , -readers , ...] @@ -88,6 +88,11 @@ def make_args_string(args_map_in): "readers": 1, "aod-memory-rate-limit": 500000000} + # get common args from analysis configuration and add to args_map + common_args_from_config = ana.get("common_args", {}) + for key, value in common_args_from_config.items(): + args_map[key] = value + # arguments dedicated for this analysis args_map_overwrite = {} @@ -98,6 +103,8 @@ def make_args_string(args_map_in): print("ERROR: Cannot digest common args.") return None + analysis_name = ana["name"] + for i in range(0, len(all_common_args), 2): tokens = all_common_args[i].split("-") key = "-".join(tokens[1:]) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index 75058219e..c50ed6999 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -251,7 +251,7 @@ def add_analysis_tasks(workflow, input_aod="./AO2D.root", output_dir="./Analysis continue print(f"INFO: Analysis {ana['name']} uses configuration {configuration}") - add_common_args_ana = get_common_args_as_string(ana["name"], add_common_args) + add_common_args_ana = get_common_args_as_string(ana, add_common_args) if not add_common_args_ana: print(f"ERROR: Cannot parse common args for analysis {ana['name']}") continue diff --git a/MC/config/analysis_testing/json/analyses_config.json b/MC/config/analysis_testing/json/analyses_config.json index 63c67b285..6be135b8b 100644 --- a/MC/config/analysis_testing/json/analyses_config.json +++ b/MC/config/analysis_testing/json/analyses_config.json @@ -233,6 +233,9 @@ "expected_output": ["AnalysisResults.root"], "valid_mc": true, "valid_data": true, + "common_args": { + "shm-segment-size": 2500000000 + }, "tasks": ["o2-analysis-je-emc-eventselection-qa", "o2-analysis-je-emc-cellmonitor", "o2-analysis-je-emcal-correction-task", From 0bb1b31c03377dc1caebf22e5e5ba75923ad2f05 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 1 Mar 2024 12:29:52 +0100 Subject: [PATCH 0412/1239] [Anchor] Remove -k from QC run, return error code (#1513) Co-authored-by: Benedikt Volkel --- MC/run/ANCHOR/anchorMC.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index 13512fea7..c1096e7ac 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -46,8 +46,8 @@ print_help() echo "ALIEN_JDL_SIMENGINE or SIMENGINE, choose the transport engine, default: TGeant4," echo "ALIEN_JDL_WORKFLOWDETECTORS, set detectors to be taken into account, default: ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP," echo "ALIEN_JDL_ANCHOR_SIM_OPTIONS, additional options that are passed to the workflow creation, default: -gen pythia8," - echo "ALIEN_JDL_ADDTIMESERIESINMC, run TPC time series. Default: 1, switch off by setting to 0," - echo "DISABLE_QC, set this to disable QC, e.g. to 1" + echo "ALIEN_JDL_ADDTIMESERIESINMC, run TPC time series. Switch off by setting to 0, default: 1," + echo "ALIEN_JDL_ANCHOR_SIM_DISABLE_QC|ANCHOR_SIM_DISABLE_QC, set this to disable QC, e.g. to 1, default: 0," } # Prevent the script from being soured to omit unexpected surprises when exit is used @@ -85,6 +85,7 @@ export ALIEN_JDL_SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE:-TGeant4}} export ALIEN_JDL_WORKFLOWDETECTORS=${ALIEN_JDL_WORKFLOWDETECTORS:-ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP} # can be passed to contain additional options that will be passed to o2dpg_sim_workflow_anchored.py and eventually to o2dpg_sim_workflow.py export ALIEN_JDL_ANCHOR_SIM_OPTIONS=${ALIEN_JDL_ANCHOR_SIM_OPTIONS:--gen pythia8} +export ALIEN_JDL_ANCHOR_SIM_DISABLE_QC=${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC:-${ANCHOR_SIM_DISABLE_QC:-0}} # all others MUST be set by the user/on the outside export ALIEN_JDL_LPMANCHORPASSNAME=${ALIEN_JDL_LPMANCHORPASSNAME:-${ANCHORPASSNAME}} export ALIEN_JDL_MCANCHOR=${ALIEN_JDL_MCANCHOR:-${MCANCHOR}} @@ -237,7 +238,7 @@ export FAIRMQ_IPC_PREFIX=./ echo "Ready to start main workflow" -${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT} MCRC=$? # <--- we'll report back this code if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then @@ -246,14 +247,12 @@ if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt tpctimes fi -[[ ! -z "${DISABLE_QC}" ]] && echo "INFO: QC is disabled, skip it." +[[ "${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC}" != "0" ]] && echo "INFO: QC is disabled, skip it." -if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then +if [[ "${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC}" == "0" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then # do QC tasks echo "Doing QC" - ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k - # NOTE that with the -k|--keep-going option, the runner will try to keep on executing even if some tasks fail. - # That means, even if there is a failing QC task, the return code will be 0 + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT} MCRC=$? fi From 928d82f57b81889dbdf0b71e55921cb4d5cfc483 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 1 Mar 2024 13:06:07 +0100 Subject: [PATCH 0413/1239] Revert "[Anchor] Remove -k from QC run, return error code (#1513)" (#1515) This reverts commit 0bb1b31c03377dc1caebf22e5e5ba75923ad2f05. --- MC/run/ANCHOR/anchorMC.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/MC/run/ANCHOR/anchorMC.sh b/MC/run/ANCHOR/anchorMC.sh index c1096e7ac..13512fea7 100755 --- a/MC/run/ANCHOR/anchorMC.sh +++ b/MC/run/ANCHOR/anchorMC.sh @@ -46,8 +46,8 @@ print_help() echo "ALIEN_JDL_SIMENGINE or SIMENGINE, choose the transport engine, default: TGeant4," echo "ALIEN_JDL_WORKFLOWDETECTORS, set detectors to be taken into account, default: ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP," echo "ALIEN_JDL_ANCHOR_SIM_OPTIONS, additional options that are passed to the workflow creation, default: -gen pythia8," - echo "ALIEN_JDL_ADDTIMESERIESINMC, run TPC time series. Switch off by setting to 0, default: 1," - echo "ALIEN_JDL_ANCHOR_SIM_DISABLE_QC|ANCHOR_SIM_DISABLE_QC, set this to disable QC, e.g. to 1, default: 0," + echo "ALIEN_JDL_ADDTIMESERIESINMC, run TPC time series. Default: 1, switch off by setting to 0," + echo "DISABLE_QC, set this to disable QC, e.g. to 1" } # Prevent the script from being soured to omit unexpected surprises when exit is used @@ -85,7 +85,6 @@ export ALIEN_JDL_SIMENGINE=${ALIEN_JDL_SIMENGINE:-${SIMENGINE:-TGeant4}} export ALIEN_JDL_WORKFLOWDETECTORS=${ALIEN_JDL_WORKFLOWDETECTORS:-ITS,TPC,TOF,FV0,FT0,FDD,MID,MFT,MCH,TRD,EMC,PHS,CPV,HMP,CTP} # can be passed to contain additional options that will be passed to o2dpg_sim_workflow_anchored.py and eventually to o2dpg_sim_workflow.py export ALIEN_JDL_ANCHOR_SIM_OPTIONS=${ALIEN_JDL_ANCHOR_SIM_OPTIONS:--gen pythia8} -export ALIEN_JDL_ANCHOR_SIM_DISABLE_QC=${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC:-${ANCHOR_SIM_DISABLE_QC:-0}} # all others MUST be set by the user/on the outside export ALIEN_JDL_LPMANCHORPASSNAME=${ALIEN_JDL_LPMANCHORPASSNAME:-${ANCHORPASSNAME}} export ALIEN_JDL_MCANCHOR=${ALIEN_JDL_MCANCHOR:-${MCANCHOR}} @@ -238,7 +237,7 @@ export FAIRMQ_IPC_PREFIX=./ echo "Ready to start main workflow" -${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT} +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt ${ALIEN_JDL_O2DPGWORKFLOWTARGET:-aod} --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} MCRC=$? # <--- we'll report back this code if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then @@ -247,12 +246,14 @@ if [[ "${ALIEN_JDL_ADDTIMESERIESINMC}" != "0" ]]; then ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt tpctimes fi -[[ "${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC}" != "0" ]] && echo "INFO: QC is disabled, skip it." +[[ ! -z "${DISABLE_QC}" ]] && echo "INFO: QC is disabled, skip it." -if [[ "${ALIEN_JDL_ANCHOR_SIM_DISABLE_QC}" == "0" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then +if [[ -z "${DISABLE_QC}" && "${MCRC}" = "0" && "${remainingargs}" == *"--include-local-qc"* ]] ; then # do QC tasks echo "Doing QC" - ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT} + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels QC --cpu-limit ${ALIEN_JDL_CPULIMIT:-8} -k + # NOTE that with the -k|--keep-going option, the runner will try to keep on executing even if some tasks fail. + # That means, even if there is a failing QC task, the return code will be 0 MCRC=$? fi From cf41e3246dcd6accdda8493135cd276b72479507 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 1 Mar 2024 18:15:09 +0100 Subject: [PATCH 0414/1239] [SimCI] Revise tests (#1517) * standalone AnalysisQC test has been remobved in favor of developments in workflow test * generators * do not blindly test all INI files when run_generator_tests.sh changes * find files correctly when there are other changed files not related to generator testing * workflows revised logic such that * if MC/bin changes --> run anchored, --> check correct creation of workflows implemented by PWGs, --> test AnalysisQC and QC * if MC/analysis_testing or MC-related QC configurations change --> test AnalysisQC and QC, --> test O2DPG AnalysisQC CLI * if anchored-related shell scripts change --> run anchored * relval * no changes Co-authored-by: Benedikt Volkel --- test/common/utils/utils.sh | 34 +++- test/run_analysisqc_tests.sh | 158 +----------------- test/run_generator_tests.sh | 48 +++--- test/run_workflow_tests.sh | 314 +++++++++++++++++------------------ 4 files changed, 207 insertions(+), 347 deletions(-) diff --git a/test/common/utils/utils.sh b/test/common/utils/utils.sh index 0c34c5395..03c01d96f 100644 --- a/test/common/utils/utils.sh +++ b/test/common/utils/utils.sh @@ -4,6 +4,33 @@ # Test utility functionality # +# a global counter for tests +TEST_COUNTER=0 + +# Prepare some colored output +SRED="\033[0;31m" +SGREEN="\033[0;32m" +SYELLOW="\033[0;33m" +SEND="\033[0m" + +echo_green() +{ + echo -e "${SGREEN}${*}${SEND}" +} + + +echo_red() +{ + echo -e "${SRED}${*}${SEND}" +} + + +echo_yellow() +{ + echo -e "${SYELLOW}${*}${SEND}" +} + + remove_artifacts() { [[ "${KEEP_ONLY_LOGS}" == "1" ]] && find . -type f ! -name '*.log' -and ! -name "*serverlog*" -and ! -name "*mergerlog*" -and ! -name "*workerlog*" -delete @@ -25,7 +52,12 @@ get_changed_files() [[ ! -z "$(git diff)" && -z ${ALIBUILD_HEAD_HASH+x} && -z ${O2DPG_TEST_HASH_HEAD+x} ]] && hash_head="" # if there are unstaged changes and no base from user, set to HEAD [[ ! -z "$(git diff)" && -z ${ALIBUILD_HEAD_HASH+x} && -z ${O2DPG_TEST_HASH_BASE+x} ]] && hash_base="HEAD" - git diff --diff-filter=AMR --name-only ${hash_base} ${hash_head} + local paths=$(git diff --diff-filter=AMR --name-only ${hash_base} ${hash_head}) + local absolute_paths= + for p in ${paths} ; do + absolute_paths+="$(realpath ${p}) " + done + echo "${absolute_paths}" } diff --git a/test/run_analysisqc_tests.sh b/test/run_analysisqc_tests.sh index bd57493fd..a4064ba30 100755 --- a/test/run_analysisqc_tests.sh +++ b/test/run_analysisqc_tests.sh @@ -1,160 +1,4 @@ #!/bin/bash -# The test parent dir to be cretaed in current directory -TEST_PARENT_DIR="o2dpg_tests/analysisqc" - -# unified names of log files -LOG_FILE="o2dpg-test-analysisqc.log" - -# Prepare some colored output -SRED="\033[0;31m" -SGREEN="\033[0;32m" -SEND="\033[0m" - - -echo_green() -{ - echo -e "${SGREEN}${*}${SEND}" -} - - -echo_red() -{ - echo -e "${SRED}${*}${SEND}" -} - - -get_git_repo_directory() -{ - local repo= - if [[ -d .git ]] ; then - pwd - else - repo=$(git rev-parse --git-dir 2> /dev/null) - fi - [[ "${repo}" != "" ]] && repo=${repo%%/.git} - echo ${repo} -} - - -test_analysisqc() -{ - echo "### Testing AnalysisQC creation for MC ###" > ${LOG_FILE} - ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root --is-mc -o wokflow_test_mc.json >> ${LOG_FILE} 2>&1 - local ret=${?} - [[ "${ret}" != "0" ]] && echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE} - echo "### Testing AnalysisQC creation for data ###" >> ${LOG_FILE} - ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root -o wokflow_test_data.json >> ${LOG_FILE} 2>&1 - local ret_data=${?} - [[ "${ret_data}" != "0" ]] && { echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE} ; ret=${ret_data} ; } - return ${ret} -} - - -print_usage() -{ - echo - echo "usage: run_workflow_tests.sh" - echo - echo " ENVIRONMENT VARIABLES:" - echo - echo " O2DPG_TEST_REPO_DIR : Point to the source repository you want to test." - echo " O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional)" - echo " O2DPG_TEST_HASH_HEAD : The head hash you want to use for comparison (optional)" - echo - echo " If O2DPG_TEST_HASH_BASE is not set, it will be looked for ALIBUILD_BASE_HASH." - echo " If also not set, this will be set to HEAD~1. However, if there are unstaged" - echo " changes, it will be set to HEAD." - echo - echo " If O2DPG_TEST_HASH_HEAD is not set, it will be looked for ALIBUILD_HEAD_HASH." - echo " If also not set, this will be set to HEAD. However, if there are unstaged" - echo " changes, it will left blank." - echo -} - -while [ "$1" != "" ] ; do - case $1 in - --help|-h ) print_usage - exit 1 - ;; - * ) echo "Unknown argument ${1}" - exit 1 - ;; - esac -done - -echo -echo "################################" -echo "# Run O2DPG AnalysisQC testing #" -echo "################################" -echo - -REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)} -if [[ ! -d ${REPO_DIR}/.git ]] ; then - echo_red "Directory \"${REPO_DIR}\" is not a git repository." - exit 1 -fi - -if [[ -z ${O2DPG_ROOT+x} ]] ; then - echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment." - exit 1 -fi - -# source the utilities -source ${REPO_DIR}/test/common/utils/utils.sh - -# Do the initial steps in the source dir where we have the full git repo -pushd ${REPO_DIR} > /dev/null - -# flag if anything changed for AnalysisQC -need_testing=$(get_changed_files | grep "MC/.*analysis_testing") - -# go back to where we came from -popd > /dev/null -REPO_DIR=$(realpath ${REPO_DIR}) - -# Now, do the trick: -# We just use the source dir since O2DPG's installation is basically just a copy of the whole repo. -# This makes sense in particular for local testing but also in the CI it works in the same way. We could do -# [[ -z {ALIBUILD_HEAD_HASH+x} ]] && export O2DPG_ROOT=${REPO_DIR} -# but let's do the same for both local and CI consistently -export O2DPG_ROOT=${REPO_DIR} - - -############### -# Let's do it # -############### -ret_global=0 -# prepare our local test directory for PWG tests -rm -rf ${TEST_PARENT_DIR} 2>/dev/null -mkdir -p ${TEST_PARENT_DIR} 2>/dev/null -pushd ${TEST_PARENT_DIR} > /dev/null - -# Test what we found -if [[ "${need_testing}" != "" ]] ; then - test_analysisqc - ret_global=${?} -else - echo "Nothing to test" - exit 0 -fi - -# return to where we came from -popd > /dev/null - -# However, if a central test fails, exit code will be !=0 -if [[ "${ret_global}" != "0" ]] ; then - echo - echo "########################" - echo "# ERROR for AnalysisQC #" - echo "########################" - echo - print_error_logs ${TEST_PARENT_DIR} - exit ${ret_global} -fi - -echo -echo_green "AnalysisQC tests successful" -echo - +# for now, obsolete exit 0 diff --git a/test/run_generator_tests.sh b/test/run_generator_tests.sh index d5a4d3c74..79addd3ea 100755 --- a/test/run_generator_tests.sh +++ b/test/run_generator_tests.sh @@ -250,35 +250,26 @@ add_ini_files_from_tests() done } -add_ini_files_from_all_tests() -{ - # Collect also those INI files for which the test has been changed - local all_tests=$(find ${REPO_DIR} -name "*.C" | grep "MC/.*/ini/tests") - local repo_dir_head=${REPO_DIR} - for t in ${all_tests} ; do - local this_test=$(realpath ${t}) - this_test=${this_test##${repo_dir_head}/} - local tc=$(basename ${this_test}) - this_test=${this_test%%/tests/*} - tc=${tc%.C}.ini - tc=${this_test}/${tc} - [[ "${INI_FILES}" == *"${tc}"* ]] && continue - INI_FILES+=" ${tc} " - done -} - collect_ini_files() { # Collect all INI files which have changed - local ini_files=$(get_changed_files | grep ".ini$" | grep "MC/config") - for ini in ${ini_files} ; do + local changed_files=$(get_changed_files) + for ini in ${changed_files} ; do + [[ "${ini}" != *"MC/config"*".ini" ]] && continue [[ "${INI_FILES}" == *"${ini}"* ]] && continue || INI_FILES+=" ${ini} " done # this relies on INI_FILES and MACRO_FILES_POTENTIALLY_INCLUDED # collect all INI files that might include some changed macros - add_ini_files_from_macros $(get_changed_files | grep ".C$" | grep "MC/config") + changed_files=$(get_changed_files) + local macros= + for m in ${changed_files} ; do + [[ "${m}" != *"MC/config"*".C" ]] && continue + macros+=" ${m} " + done + + add_ini_files_from_macros ${macros} # this relies on MACRO_FILES_POTENTIALLY_INCLUDED # collect all INI files that might contain macros which in turn include changed macros @@ -286,7 +277,13 @@ collect_ini_files() add_ini_files_from_macros $(find_including_macros) # also tests might have changed in which case we run them - add_ini_files_from_tests $(get_changed_files | grep ".C$" | grep "MC/.*/ini/tests") + changed_files=$(get_changed_files) + local macros= + for m in ${changed_files} ; do + [[ "${m}" != *"MC/"*"ini/tests"*".C" ]] && continue + macros+=" ${m} " + done + add_ini_files_from_tests ${macros} } @@ -361,12 +358,12 @@ echo REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)} if [[ ! -d ${REPO_DIR}/.git ]] ; then - echo_red "Directory \"${REPO_DIR}\" is not a git repository." + echo "ERROR: Directory \"${REPO_DIR}\" is not a git repository." exit 1 fi if [[ -z ${O2DPG_ROOT+x} ]] ; then - echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment." + echo "ERROR: O2DPG is not loaded, probably other packages are missing as well in this environment." exit 1 fi @@ -376,11 +373,6 @@ source ${REPO_DIR}/test/common/utils/utils.sh # Do the initial steps in the source dir where we have the full git repo pushd ${REPO_DIR} > /dev/null -# First check, if testing itself has changed. In that case this will add INI files -# for which a test can be found -global_testing_changed=$(get_changed_files | grep -E "common/kine_tests/test_generic_kine.C|run_generator_tests.sh" | grep "^test/") -[[ "${global_testing_changed}" != "" ]] && add_ini_files_from_all_tests - # Then add the ini files that have changed as well. We need to do that so we get information # about missing tests etc. collect_ini_files diff --git a/test/run_workflow_tests.sh b/test/run_workflow_tests.sh index 2acde102e..9962b3293 100755 --- a/test/run_workflow_tests.sh +++ b/test/run_workflow_tests.sh @@ -2,56 +2,30 @@ # The test parent dir to be cretaed in current directory TEST_PARENT_DIR_PWG="o2dpg_tests/workflows_pwgs" -TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_bin" +TEST_PARENT_DIR_BIN="o2dpg_tests/workflows_analysisqc" TEST_PARENT_DIR_ANCHORED="o2dpg_tests/anchored" -# a global counter for tests -TEST_COUNTER=0 - # unified names of log files LOG_FILE_WF="o2dpg-test-wf.log" LOG_FILE_ANCHORED="o2dpg-test-anchored.log" - -# Prepare some colored output -SRED="\033[0;31m" -SGREEN="\033[0;32m" -SEND="\033[0m" - - -echo_green() -{ - echo -e "${SGREEN}${*}${SEND}" -} +LOG_FILE_ANALYSISQC="o2dpg-test_analysisqc.log" -echo_red() -{ - echo -e "${SRED}${*}${SEND}" -} - get_git_repo_directory() { + local look_dir=${1:-$(pwd)} + look_dir=$(realpath "${look_dir}") + look_dir=${look_dir%%/.git} local repo= - if [[ -d .git ]] ; then - pwd - else + ( + cd "${look_dir}" repo=$(git rev-parse --git-dir 2> /dev/null) - fi - [[ "${repo}" != "" ]] && repo=${repo%%/.git} + [[ "${repo}" != "" ]] && { repo=$(realpath "${repo}") ; repo=${repo%%/.git} ; } + ) echo ${repo} } -get_all_workflows() -{ - # Collect also those INI files for which the test has been changed - local repo_dir_head=${REPO_DIR} - local grep_dir=${1} - local all_workflows=$(find ${repo_dir_head} -name "*.sh" | grep "${grep_dir}") - echo ${all_workflows} -} - - test_single_wf() { local wf_script=${1} @@ -59,12 +33,12 @@ test_single_wf() make_wf_creation_script ${wf_script} ${wf_script_local} local has_wf_script_local=${?} echo -n "Test ${TEST_COUNTER}: ${wfs}" - [[ "${has_wf_script_local}" != "0" ]] && { echo "No WF creation in script ${wfs} ##########" ; return 1 ; } + [[ "${has_wf_script_local}" != "0" ]] && { echo -n " (No WF creation in script)" ; echo_red " -> FAILED" ; return 1 ; } # Check if there is an "exit" other than the usual # [ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 # like ones. # This is not perfect but might prevent us from running into some checks the WF script does before launching the WF creation - [[ "$(grep exit ${wf_script_local} | grep -v "This needs")" != "" ]] && { echo -e -n "\nFound \"exit\" in ${wfs} so will not test automatically" ; return 0 ; } + [[ "$(grep exit ${wf_script_local} | grep -v "This needs")" != "" ]] && { echo -n " (Found \"exit\" in script, not testing automatically)" ; echo_yellow " -> WARNING" ; return 0 ; } # one single test echo "Test ${wf_line} from ${wfs}" > ${LOG_FILE_WF} bash ${wf_script_local} >> ${LOG_FILE_WF} 2>&1 @@ -72,6 +46,7 @@ test_single_wf() local ret_this_qc=0 local ret_this_analysis=0 if [[ "${ret_this}" != "0" ]] ; then + echo_red " -> FAILED" echo "[FATAL]: O2DPG_TEST Workflow creation failed" >> ${LOG_FILE_WF} elif [[ "${execute}" != "" ]] ; then local memlimit=${O2DPG_TEST_WORKFLOW_MEMLIMIT:+--mem-limit ${O2DPG_TEST_WORKFLOW_MEMLIMIT}} @@ -80,7 +55,9 @@ test_single_wf() [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels QC ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_qc=${?} ; } [[ "${ret_this}" == "0" ]] && { ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --cpu-limit 8 --target-labels Analysis ${memlimit} >> ${LOG_FILE_WF} 2>&1 ; ret_this_analysis=${?} ; } ret_this=$((ret_this + ret_this_qc + ret_this_analysis)) - [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} + [[ "${ret_this}" != "0" ]] && echo "[FATAL]: O2DPG_TEST Workflow execution failed" >> ${LOG_FILE_WF} || echo_green " -> PASSED" + else + echo_green " -> PASSED" fi return ${ret_this} } @@ -91,8 +68,8 @@ run_workflow_creation() local execute= while [ "$1" != "" ] ; do case $1 in - --execute ) shift - execute=1 + --execute ) execute=1 + shift ;; * ) wf_scripts+="${1} " shift @@ -116,16 +93,34 @@ run_workflow_creation() local ret_this=${?} [[ "${ret_this}" != "0" ]] && RET=${ret_this} popd > /dev/null - if [[ "${ret_this}" != "0" ]] ; then - echo_red " -> FAILED" - else - echo_green " -> PASSED" - fi done return ${RET} } + +test_analysisqc_cli() +{ + ((TEST_COUNTER++)) + local test_dir="${TEST_COUNTER}_analysisqc_cli" + rm -rf ${test_dir} 2> /dev/null + mkdir ${test_dir} + pushd ${test_dir} > /dev/null + echo "### Testing AnalysisQC creation for MC ###" > ${LOG_FILE_ANALYSISQC} + echo -n "Test ${TEST_COUNTER}: Running AnalysisQC CLI" + ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root --is-mc -o wokflow_test_mc.json >> ${LOG_FILE_ANALYSISQC} 2>&1 + local ret=${?} + [[ "${ret}" != "0" ]] && echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE_ANALYSISQC} + echo "### Testing AnalysisQC creation for data ###" >> ${LOG_FILE_ANALYSISQC} + ${O2DPG_ROOT}/MC/analysis_testing/o2dpg_analysis_test_workflow.py -f AO2D.root -o wokflow_test_data.json >> ${LOG_FILE_ANALYSISQC} 2>&1 + local ret_data=${?} + [[ "${ret_data}" != "0" ]] && { echo "[FATAL]: O2DPG_TEST failed" >> ${LOG_FILE_ANALYSISQC} ; ret=${ret_data} ; } + popd > /dev/null + [[ "${ret}" != "0" ]] && echo_red " -> FAILED" || echo_green " -> PASSED" + return ${ret} +} + + test_anchored() { local to_run="${1:-${O2DPG_ROOT}/MC/run/ANCHOR/tests/test_anchor_2023_apass2_pp.sh}" @@ -140,27 +135,24 @@ test_anchored() echo -n "Test ${TEST_COUNTER}: ${anchored_script}" ${anchored_script} >> ${LOG_FILE_ANCHORED} 2>&1 local ret_this=${?} - [[ "${ret_this}" != "0" ]] && RET=${ret_this} + if [[ "${ret_this}" != "0" ]] ; then + echo_red " -> FAILED" + RET=${ret_this} + else + echo_green " -> PASSED" + fi popd > /dev/null done return ${RET} } -collect_changed_pwg_wf_files() -{ - # Collect all INI files which have changed - local wf_scripts=$(get_changed_files | grep ".sh$" | grep "MC/run") - for wfs in ${wf_scripts} ; do - [[ "${WF_FILES}" == *"${wfs}"* ]] && continue || WF_FILES+=" ${wfs} " - done -} - print_usage() { + echo echo "usage: run_workflow_tests.sh" echo - echo " ENVIRONMENT VARIABLES:" + echo " ENVIRONMENT VARIABLES TO DETERMINE WHAT TO COMPARE:" echo echo " O2DPG_TEST_REPO_DIR : Point to the source repository you want to test." echo " O2DPG_TEST_HASH_BASE : The base hash you want to use for comparison (optional)" @@ -174,10 +166,15 @@ print_usage() echo " If also not set, this will be set to HEAD. However, if there are unstaged" echo " changes, it will left blank." echo + echo " SPECIFIC ENVIRONMENT VARIABLES FOR THIS TEST:" echo " O2DPG_TEST_WORKFLOW_MEMLIMIT : The memory limit that is passed to the workflow runner in case a workflow is executed (optional)" echo } + +############# +# Main part # +############# while [ "$1" != "" ] ; do case $1 in --help|-h ) print_usage @@ -189,47 +186,70 @@ while [ "$1" != "" ] ; do esac done -echo -echo "##############################" -echo "# Run O2DPG workflow testing #" -echo "##############################" -echo - +# determine the repository directory REPO_DIR=${O2DPG_TEST_REPO_DIR:-$(get_git_repo_directory)} if [[ ! -d ${REPO_DIR}/.git ]] ; then - echo_red "Directory \"${REPO_DIR}\" is not a git repository." + echo "ERROR: Directory \"${REPO_DIR}\" is not a git repository." exit 1 fi if [[ -z ${O2DPG_ROOT+x} ]] ; then - echo_red "O2DPG is not loaded, probably other packages are missing as well in this environment." + echo "ERROR: O2DPG is not loaded, probably other packages are missing as well in this environment." exit 1 fi # source the utilities source ${REPO_DIR}/test/common/utils/utils.sh + +echo "##############################" +echo "# Run O2DPG workflow testing #" +echo "##############################" + # Do the initial steps in the source dir where we have the full git repo pushd ${REPO_DIR} > /dev/null # flag if anything changed in the sim workflow bin dir -changed_wf_bin=$(get_changed_files | grep -E "MC/bin") -changed_wf_bin_related=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json") -changed_anchored_related=$(get_changed_files | grep -E "MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py") - +changed_sim_bin=$(get_changed_files | grep -E "MC/bin") +# collect if anything has changed related to AnalysisQC +changed_analysis_qc=$(get_changed_files | grep -E "MC/analysis_testing|MC/config/analysis_testing/json|MC/config/QC/json") +# check if anything has changed concerning anchoring +changed_anchored=$(get_changed_files | grep -E "MC/bin|MC/run/ANCHOR/anchorMC.sh|MC/run/ANCHOR/tests|MC/bin|UTILS/parse-async-WorkflowConfig.py|DATA/production/configurations/asyncReco/setenv_extra.sh|DATA/production/configurations/asyncReco/async_pass.sh|DATA/common/setenv.sh|DATA/production/workflow-multiplicities.sh") +# collect changed workflow scripts +changed_workflows= +# workflows to be executed +execute_workflows= +echo "==> Test outline" +if [[ "${changed_sim_bin}" != "" ]] ; then + # in this case, something central has changed, test creation of all workflows against it + echo " - The creation of simulation workflows from all run scripts (MC/run/**/*.sh) will be tested." + for p in $(find MC/run -name "*.sh") ; do + changed_workflows+="$(realpath ${p}) " + done + # definitely run anchored if central python scripts have changed + echo " - Changes in MC/bin/ detected, mark anchored MC test to be run." + changed_anchored="1" +else + # otherwise, only take the changed shell scripts + changed_workflows= + changed_files=$(get_changed_files) + for cf in ${changed_files} ; do + [[ "${cf}" != *"MC/run"*".sh" ]] && continue + changed_workflows+="${cf} " + done + [[ "${changed_workflows}" != "" ]] && echo " - The creation of simulation workflows from changed run scripts (sub-sect of MC/run/**/*.sh) will be tested." +fi -# collect what has changed for PWGs -collect_changed_pwg_wf_files +if [[ "${changed_analysis_qc}" != "" || "${changed_sim_bin}" ]] ; then + for p in $(find "MC/bin/tests" -name "*.sh") ; do + execute_workflows+="$(realpath ${p}) " + done + echo " - Test AnalysisQC CLI and execution with a simulation." +fi -# get realpaths for all changes -wf_files_tmp=${WF_FILES} -WF_FILES= -for wf_tmp in ${wf_files_tmp} ; do - # convert to full path so that we can find it from anywhere - WF_FILES+="$(realpath ${wf_tmp}) " -done +[[ "${changed_anchored}" != "" ]] && echo " - Test anchored simulation." -# go back to where we came from +# everything collected, go back to where we came from popd > /dev/null REPO_DIR=$(realpath ${REPO_DIR}) @@ -241,111 +261,83 @@ REPO_DIR=$(realpath ${REPO_DIR}) export O2DPG_ROOT=${REPO_DIR} -############### -# ANCHORED MC # -############### -# prepare our local test directory for PWG tests -rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null -mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null -pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null - -# global return code for PWGs -ret_global_anchored=0 -if [[ "${changed_anchored_related}" != "" ]] ; then - echo "### Test anchored ###" - # Run an anchored test - test_anchored - ret_global_anchored=${?} - echo -fi - -# return to where we came from -popd > /dev/null - -######## -# PWGs # -######## -# prepare our local test directory for PWG tests -rm -rf ${TEST_PARENT_DIR_PWG} 2>/dev/null -mkdir -p ${TEST_PARENT_DIR_PWG} 2>/dev/null -pushd ${TEST_PARENT_DIR_PWG} > /dev/null - +############################## +# PWG workflow shell scripts # +############################## # global return code for PWGs ret_global_pwg=0 -if [[ "${changed_wf_bin}" != "" ]] ; then - # Run all the PWG related WF creations, hence overwrite what was collected by collect_changed_pwg_wf_files earlier - WF_FILES=$(get_all_workflows "MC/run/.*/") - echo -fi # Test what we found -if [[ "${WF_FILES}" != "" ]] ; then - echo "### Test PWG-related workflow creation ###" +if [[ "${changed_workflows}" != "" ]] ; then + # prepare our local test directory for PWG tests + rm -rf ${TEST_PARENT_DIR_PWG} 2>/dev/null + mkdir -p ${TEST_PARENT_DIR_PWG} 2>/dev/null + pushd ${TEST_PARENT_DIR_PWG} > /dev/null + echo - run_workflow_creation ${WF_FILES} + echo "==> START BLOCK: Test PWG-related workflow creation <==" + run_workflow_creation ${changed_workflows} ret_global_pwg=${?} - echo + [[ "${ret_global_pwg}" != "0" ]] && { echo "WARNING for workflows creations, some could not be built." ; print_error_logs ./ ; } + echo "==> END BLOCK: Test PWG-related workflow creation <==" + + # return to where we came from + popd > /dev/null fi -# return to where we came from -popd > /dev/null -#################### -# sim workflow bin # -#################### +#################################### +# sim workflow bin with AnalysisQC # +#################################### # prepare our local test directory for bin tests -rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null -mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null -pushd ${TEST_PARENT_DIR_BIN} > /dev/null - # global return code for PWGs -ret_global_bin=0 -if [[ "${changed_wf_bin}" != "" || "${changed_wf_bin_related}" != "" ]] ; then - echo "### Test bin-related workflow creation ###" +ret_analysis_qc=0 +if [[ "${changed_analysis_qc}" != "" ]] ; then + rm -rf ${TEST_PARENT_DIR_BIN} 2>/dev/null + mkdir -p ${TEST_PARENT_DIR_BIN} 2>/dev/null + pushd ${TEST_PARENT_DIR_BIN} > /dev/null + echo + echo "==> START BLOCK: Test running workflow with AnalysisQC <==" + # test command line interface + test_analysisqc_cli + ret_analysis_qc=${?} # Run all the bin test WF creations - run_workflow_creation $(get_all_workflows "MC/bin/tests") --execute - ret_global_bin=${?} - echo + [[ "${ret_analysis_qc}" == "0" ]] && { run_workflow_creation ${execute_workflows} --execute ; ret_analysis_qc=${?} ; } + [[ "${ret_analysis_qc}" != "0" ]] && { echo "ERROR for workflows execution and AnalysisQC." ; print_error_logs ./ ; } + echo "==> END BLOCK: Test running workflow with AnalysisQC <==" + + # return to where we came from + popd > /dev/null fi -# return to where we came from -popd > /dev/null -# final printing of log files of failed tests -# For PWG workflows, this triggers only a warning at the moment -if [[ "${ret_global_pwg}" != "0" ]] ; then - echo - echo "#####################################" - echo "# WARNING for PWG-related workflows #" - echo "#####################################" - echo - print_error_logs ${TEST_PARENT_DIR_PWG} -fi +############### +# ANCHORED MC # +############### +# global return code for PWGs +ret_global_anchored=0 +if [[ "${changed_anchored}" != "" ]] ; then + # prepare our local test directory for PWG tests + rm -rf ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null + mkdir -p ${TEST_PARENT_DIR_ANCHORED} 2>/dev/null + pushd ${TEST_PARENT_DIR_ANCHORED} > /dev/null -# However, if a central test fails, exit code will be !=0 -if [[ "${ret_global_bin}" != "0" ]] ; then - echo - echo "###################################" - echo "# ERROR for bin-related workflows #" - echo "###################################" echo - print_error_logs ${TEST_PARENT_DIR_BIN} -fi + echo "==> START BLOCK: Test anchored simulation" + # Run an anchored test + test_anchored + ret_global_anchored=${?} + [[ "${ret_global_anchored}" != "0" ]] && { echo "ERROR executing anchored simulation." ; print_error_logs ./ ; } + echo "==> END BLOCK: Test anchored simulation" -# However, if a central test fails, exit code will be !=0 -if [[ "${ret_global_anchored}" != "0" ]] ; then - echo - echo "##########################" - echo "# ERROR for anchored MCs #" - echo "##########################" - echo - print_error_logs ${TEST_PARENT_DIR_ANCHORED} + # return to where we came from + popd > /dev/null fi -RET=$(( ret_global_bin + ret_global_anchored )) +RET=$(( ret_analysis_qc + ret_global_anchored )) echo -[[ "${RET}" != "0" ]] && echo "There were errors, please check!" || echo_green "All required workflow tests successful" +[[ "${RET}" != "0" ]] && echo_red "There were errors, please check!" || echo_green "All required workflow tests successful" exit ${RET} From 77e6a0613fae0c61f33d65d00b316f9bd2f2d54d Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 1 Mar 2024 14:36:01 +0100 Subject: [PATCH 0415/1239] Possibility to take QC alone when we split the wf --- .../configurations/asyncReco/async_pass.sh | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index bd6e9ce89..8f1e9e653 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -575,11 +575,17 @@ else fi if ([[ -z "$ALIEN_JDL_SSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 3 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 3 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq "all" ]]; then - # 3. matching, QC, calib, AOD + # 3. matching, calib, AOD, potentially QC WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS_START + if [[ "$ALIEN_JDL_KEEPQCSEPARATE" == "1" ]]; then + echo "QC will be run as last step, removing it from 3rd step" + for i in QC; do + export WORKFLOW_PARAMETERS=$(echo $WORKFLOW_PARAMETERS | sed -e "s/,$i,/,/g" -e "s/^$i,//" -e "s/,$i"'$'"//" -e "s/^$i"'$'"//") + done + fi echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" - echo "Step 3) matching, QC, calib, AOD" - echo -e "\nStep 3) matching, QC, calib, AOD" >> workflowconfig.log + echo "Step 3) matching, calib, AOD, potentially QC" + echo -e "\nStep 3) matching, calib, AOD, potentially QC" >> workflowconfig.log export TIMEFRAME_RATE_LIMIT=0 echo "Removing detectors $DETECTORS_EXCLUDE" READER_DELAY=${ALIEN_JDL_READERDELAY:-30} @@ -607,13 +613,45 @@ else fi fi fi + if [[ "$ALIEN_JDL_KEEPQCSEPARATE" == "1" ]]; then + if ([[ -z "$ALIEN_JDL_SSPLITSTEP" ]] && [[ -z "$ALIEN_JDL_SSPLITSTEP" ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq 4 ]] || ( [[ -n $ALIEN_JDL_STARTSPLITSTEP ]] && [[ "$ALIEN_JDL_STARTSPLITSTEP" -le 4 ]]) || [[ "$ALIEN_JDL_SSPLITSTEP" -eq "all" ]]; then + # 4. QC + WORKFLOW_PARAMETERS="QC" + echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" + echo "Step 4) QC" + echo -e "\nStep 4) QC" >> workflowconfig.log + export TIMEFRAME_RATE_LIMIT=0 + echo "Removing detectors $DETECTORS_EXCLUDE" + env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log + # run it + if [[ "0$RUN_WORKFLOW" != "00" ]]; then + timeStart=`date +%s` + time env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=run TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list + exitcode=$? + timeEnd=`date +%s` + timeUsed=$(( $timeUsed+$timeEnd-$timeStart )) + delta=$(( $timeEnd-$timeStart )) + echo "Time spent in running the workflow, Step 4 = $delta s" + echo "exitcode = $exitcode" + if [[ $exitcode -ne 0 ]]; then + echo "exit code from Step 4 of processing is " $exitcode > validation_error.message + echo "exit code from Step 4 of processing is " $exitcode + exit $exitcode + fi + mv latest.log latest_reco_4.log + if [[ -f performanceMetrics.json ]]; then + mv performanceMetrics.json performanceMetrics_4.json + fi + fi + fi + fi fi # now extract all performance metrics if [[ $ALIEN_JDL_EXTRACTMETRICS == "1" ]]; then IFS=$'\n' timeStart=`date +%s` - for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json ; do + for perfMetricsFiles in performanceMetrics.json performanceMetrics_1.json performanceMetrics_2.json performanceMetrics_3.json performanceMetrics_4.json ; do suffix=`echo $perfMetricsFiles | sed 's/performanceMetrics\(.*\).json/\1/'` if [[ -f "performanceMetrics.json" ]]; then for workflow in `grep ': {' $perfMetricsFiles`; do From c95a0484615cc6970332498d1d437f40ca84c173 Mon Sep 17 00:00:00 2001 From: Jeremy Wilkinson Date: Sat, 2 Mar 2024 10:11:30 +0100 Subject: [PATCH 0416/1239] Fix converter logic for o2-analysis-v0converter in test workflow (#1493) * fix converter logic for o2v0converter in test workflow * add ft0-corrected-table --- MC/analysis_testing/o2dpg_analysis_test_workflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index c50ed6999..de50128ed 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -190,8 +190,9 @@ def get_additional_workflows(input_aod): o2_analysis_converters = {"O2collision_001": "o2-analysis-collision-converter --doNotSwap", "O2zdc_001": "o2-analysis-zdc-converter", "O2bc_001": "o2-analysis-bc-converter", - "O2v0_001": "o2-analysis-v0converter", - "O2trackextra_001": "o2-analysis-tracks-extra-converter"} + "O2v0_002": "o2-analysis-v0converter", + "O2trackextra_001": "o2-analysis-tracks-extra-converter", + "O2ft0corrected": "o2-analysis-ft0-corrected-table"} for i in froot.GetListOfKeys(): if "DF_" not in i.GetName(): continue From 3d0840e5094d8ed12cc73b586aa91a6b05970a1e Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 4 Mar 2024 12:13:49 +0100 Subject: [PATCH 0417/1239] [SimWF] Use __global_init_task__ more consistently (#1518) * centralise function that creates the task * apply also when using AnalysisQC CLI Co-authored-by: Benedikt Volkel --- .../o2dpg_analysis_test_workflow.py | 8 +++-- MC/bin/o2dpg_sim_workflow.py | 18 ++-------- MC/bin/o2dpg_workflow_utils.py | 33 +++++++++++++++++-- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/MC/analysis_testing/o2dpg_analysis_test_workflow.py b/MC/analysis_testing/o2dpg_analysis_test_workflow.py index de50128ed..2bcd2038a 100755 --- a/MC/analysis_testing/o2dpg_analysis_test_workflow.py +++ b/MC/analysis_testing/o2dpg_analysis_test_workflow.py @@ -80,7 +80,7 @@ o2dpg_workflow_utils = importlib.util.module_from_spec(spec) sys.modules[module_name] = o2dpg_workflow_utils spec.loader.exec_module(o2dpg_workflow_utils) -from o2dpg_workflow_utils import createTask, dump_workflow +from o2dpg_workflow_utils import createTask, dump_workflow, createGlobalInitTask module_name = "o2dpg_analysis_test_utils" spec = importlib.util.spec_from_file_location(module_name, join(O2DPG_ROOT, "MC", "analysis_testing", "o2dpg_analysis_test_utils.py")) @@ -322,7 +322,9 @@ def run(args): print("ERROR: QC upload was requested, however in that case a --pass-name and --period-name are required") return 1 - workflow = [] + ### setup global environment variables which are valid for all tasks, set as first task + global_env = {"ALICEO2_CCDB_CONDITION_NOT_AFTER": args.condition_not_after} if args.condition_not_after else None + workflow = [createGlobalInitTask(global_env)] add_analysis_tasks(workflow, args.input_file, expanduser(args.analysis_dir), is_mc=args.is_mc, analyses_only=args.only_analyses, autoset_converters=args.autoset_converters, include_disabled_analyses=args.include_disabled, timeout=args.timeout, collision_system=args.collision_system, add_common_args=args.add_common_args) if args.with_qc_upload: add_analysis_qc_upload_tasks(workflow, args.period_name, args.run_number, args.pass_name) @@ -350,6 +352,8 @@ def main(): parser.add_argument("--timeout", type=int, default=None, help="Timeout for analysis tasks in seconds.") parser.add_argument("--collision-system", dest="collision_system", help="Set the collision system. If not set, tried to be derived from ALIEN_JDL_LPMInterationType. Fallback to pp") parser.add_argument("--add-common-args", dest="add_common_args", nargs="*", help="Pass additional common arguments per analysis, for instance --add-common-args EMCAL-shm-segment-size 2500000000 will add --shm-segment-size 2500000000 to the EMCAL analysis") + parser.add_argument('--condition-not-after', dest="condition_not_after", type=int, help="only consider CCDB objects not created after this timestamp (for TimeMachine)", default=3385078236000) + parser.set_defaults(func=run) args = parser.parse_args() return(args.func(args)) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index d0812d42c..31d2ad06a 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -20,7 +20,7 @@ import sys import importlib.util import argparse -from os import environ, mkdir, getcwd +from os import environ, mkdir from os.path import join, dirname, isdir, isabs import random import json @@ -326,20 +326,8 @@ def extractVertexArgs(configKeyValuesStr, finalDiamondDict): workflow['stages'] = [] ### setup global environment variables which are valid for all tasks -globalenv = {} -if args.condition_not_after: - # this is for the time-machine CCDB mechanism - globalenv['ALICEO2_CCDB_CONDITION_NOT_AFTER'] = args.condition_not_after - # this is enforcing the use of local CCDB caching - if environ.get('ALICEO2_CCDB_LOCALCACHE') == None: - print ("ALICEO2_CCDB_LOCALCACHE not set; setting to default " + getcwd() + '/ccdb') - globalenv['ALICEO2_CCDB_LOCALCACHE'] = getcwd() + "/ccdb" - else: - # fixes the workflow to use and remember externally provided path - globalenv['ALICEO2_CCDB_LOCALCACHE'] = environ.get('ALICEO2_CCDB_LOCALCACHE') - globalenv['IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE'] = '${ALICEO2_CCDB_LOCALCACHE:+"ON"}' - -globalinittask = createGlobalInitTask(globalenv) +global_env = {'ALICEO2_CCDB_CONDITION_NOT_AFTER': args.condition_not_after} if args.condition_not_after else None +globalinittask = createGlobalInitTask(global_env) globalinittask['cmd'] = 'o2-ccdb-cleansemaphores -p ${ALICEO2_CCDB_LOCALCACHE}' workflow['stages'].append(globalinittask) #### diff --git a/MC/bin/o2dpg_workflow_utils.py b/MC/bin/o2dpg_workflow_utils.py index 748129de2..18fd600c9 100755 --- a/MC/bin/o2dpg_workflow_utils.py +++ b/MC/bin/o2dpg_workflow_utils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from os import environ, getcwd from copy import deepcopy import json @@ -84,18 +85,44 @@ def createTask(name='', needs=[], tf=-1, cwd='./', lab=[], cpu=1, relative_cpu=N 'cwd' : cwd } -def createGlobalInitTask(envdict): +def createGlobalInitTask(keys_values=None, set_defaults=True): """Returns a special task that is recognized by the executor as a task whose environment section is to be globally applied to all tasks of a workflow. - envdict: dictionary of environment variables and values to be globally applied to all tasks + Args: + keys_values: dict or None + dictionary of environment variables and values to be globally applied to all tasks + if sharing keys with defaults, keys_values takes precedence + set_defaults: bool + whether or not some default values will be added + + Returns: + dict: task dictionary """ + + # dictionary holding global environment to be passed to task + env_dict = {} + + if set_defaults: + if environ.get('ALICEO2_CCDB_LOCALCACHE') is None: + print ("ALICEO2_CCDB_LOCALCACHE not set; setting to default " + getcwd() + '/ccdb') + env_dict['ALICEO2_CCDB_LOCALCACHE'] = getcwd() + "/ccdb" + else: + # fixes the workflow to use and remember externally provided path + env_dict['ALICEO2_CCDB_LOCALCACHE'] = environ.get('ALICEO2_CCDB_LOCALCACHE') + env_dict['IGNORE_VALIDITYCHECK_OF_CCDB_LOCALCACHE'] = '${ALICEO2_CCDB_LOCALCACHE:+"ON"}' + + if keys_values: + # keys_values takes priority in case of same keys + env_dict |= keys_values + t = createTask(name = '__global_init_task__') t['cmd'] = 'NO-COMMAND' - t['env'] = envdict + t['env'] = env_dict return t + def summary_workflow(workflow): print("=== WORKFLOW SUMMARY ===\n") print(f"-> There are {len(workflow)} tasks") From 826526fdecf78c66359eba4f3fa3ef4c1cdc6973 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Mon, 4 Mar 2024 15:29:35 +0100 Subject: [PATCH 0418/1239] Run only analyses, no QCDB upload (#1519) Achieved by changing `-tt Analysis_` to `--target-labels Analysis`. Upload tasks to not have that label, instead they can be triggered with `--target-labels AnalysisUpload` --- MC/run/examples/O2DPG_pp_minbias.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/run/examples/O2DPG_pp_minbias.sh b/MC/run/examples/O2DPG_pp_minbias.sh index be23c9d80..f9b2a99c7 100755 --- a/MC/run/examples/O2DPG_pp_minbias.sh +++ b/MC/run/examples/O2DPG_pp_minbias.sh @@ -58,7 +58,7 @@ fi RETANA=0 if [ "${DOANALYSIS}" != "" ] && [ "${RETMC}" = "0" ]; then # run test analyses if requested - ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -tt "Analysis_" ${MEMLIMIT} ${CPULIMIT} + ${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json --target-labels Analysis ${MEMLIMIT} ${CPULIMIT} RETANA=${?} fi From 52c6168d3a6162e4a4e5eb9b932e2897446f0060 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Wed, 6 Mar 2024 12:52:31 +0100 Subject: [PATCH 0419/1239] Possibility to set a rate limiting from JDL for split WF --- DATA/production/configurations/asyncReco/async_pass.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 8f1e9e653..17d502b01 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -586,7 +586,7 @@ else echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 3) matching, calib, AOD, potentially QC" echo -e "\nStep 3) matching, calib, AOD, potentially QC" >> workflowconfig.log - export TIMEFRAME_RATE_LIMIT=0 + export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-0} echo "Removing detectors $DETECTORS_EXCLUDE" READER_DELAY=${ALIEN_JDL_READERDELAY:-30} export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " @@ -620,7 +620,7 @@ else echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 4) QC" echo -e "\nStep 4) QC" >> workflowconfig.log - export TIMEFRAME_RATE_LIMIT=0 + export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-0} echo "Removing detectors $DETECTORS_EXCLUDE" env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it From 97008600b2735504012173a1983adc76ceaa81ed Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Wed, 6 Mar 2024 14:10:57 +0100 Subject: [PATCH 0420/1239] stability fixes in runGRIDContainerized.sh --- GRID/utils/runGRIDContainerized.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GRID/utils/runGRIDContainerized.sh b/GRID/utils/runGRIDContainerized.sh index 1752f692d..cb57a7a1c 100755 --- a/GRID/utils/runGRIDContainerized.sh +++ b/GRID/utils/runGRIDContainerized.sh @@ -9,7 +9,7 @@ echo "Trying to run script ${SCRIPT} in a container environment" # detect architecture (ARM or X86) ARCH=$(uname -i) -if [ "$ARCH" == "aarch64" ] || [ "$arch" == "x86" ]; then +if [ "$ARCH" == "aarch64" ] || [ "$ARCH" == "x86_64" ]; then echo "Detected hardware architecture : $ARCH" else echo "Invalid architecture ${ARCH} detected. Exiting" @@ -35,9 +35,9 @@ fi # copy script to WORK_DIR cp ${SCRIPT} ${WORK_DIR}/job.sh -# export certificates (need to be created before) -ALIEN_CERTFILE=$(ls -t /tmp/tokencert_*.pem 2> /dev/null | head -n 1) -ALIEN_KEYFILE=$(ls -t /tmp/tokenkey_*.pem 2> /dev/null | head -n 1) +# export certificates - belonging to current user (need to be created before) +ALIEN_CERTFILE=$(find /tmp -type f -name 'tokencert*pem' -user `whoami` 2> /dev/null) +ALIEN_KEYFILE=$(find /tmp -type f -name 'tokenkey*pem' -user `whoami` 2> /dev/null) [ "${ALIEN_CERTFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 [ "${ALIEN_KEYFILE}" == "" ] && echo "No certificate file found; Initialize a token with alien-init-token or similar" && exit 1 From 73fe052ce6d44c5d73eb6f035ff9ac318a805426 Mon Sep 17 00:00:00 2001 From: Timo Wilken Date: Wed, 6 Mar 2024 14:22:54 +0100 Subject: [PATCH 0421/1239] Fix typo in CODEOWNERS (#1525) @aphecetche's username was misspelt. Also added @aphecetche with write permissions -- otherwise code ownership doesn't work. --- CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index f697dcaf8..27ea257cd 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,9 +8,9 @@ /DATA/testing/detectors/FV0 /DATA/testing/detectors/HMP /DATA/testing/detectors/ITS -/DATA/testing/detectors/MCH @aphecethce +/DATA/testing/detectors/MCH @aphecetche /DATA/testing/detectors/MFT -/DATA/testing/detectors/MID @aphecethce +/DATA/testing/detectors/MID @aphecetche /DATA/testing/detectors/PHS /DATA/testing/detectors/TOF @noferini @chiarazampolli /DATA/testing/detectors/TPC @wiechula From 10b933ce4b9924246723ce8ac04434584f4cfc34 Mon Sep 17 00:00:00 2001 From: Nasir Mehdi Malik <89008506+nasirmehdimalik@users.noreply.github.com> Date: Thu, 7 Mar 2024 19:27:58 +0530 Subject: [PATCH 0422/1239] TRD: add tracking QC task for mc workflow (#1467) * TRD;) tracking task in mc workflow * rmove digit query from tracklet task * removed not maxNumberCycles, applicable items * trd-digit-task name change to trd-standalone-task * Update MC/config/QC/json/trd-tracking-task.json Co-authored-by: Ole Schmidt * Update MC/config/QC/json/trd-tracking-task.json * readerCommand configured with TPC_TRD --------- Co-authored-by: Ole Schmidt --- MC/bin/o2dpg_qc_finalization_workflow.py | 3 +- MC/bin/o2dpg_sim_workflow.py | 7 ++- ...its-task.json => trd-standalone-task.json} | 23 ++++++---- MC/config/QC/json/trd-tracking-task.json | 44 +++++++++++++++++++ 4 files changed, 67 insertions(+), 10 deletions(-) rename MC/config/QC/json/{trd-digits-task.json => trd-standalone-task.json} (66%) create mode 100644 MC/config/QC/json/trd-tracking-task.json diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 373989f20..6908b4956 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -83,7 +83,8 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('emcBCQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json') add_QC_finalization('tpcStandardQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-standard-direct.json') - add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-digits-task.json') + add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json') + add_QC_finalization('trdTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json') add_QC_finalization('vertexQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/vertexing-qc-direct-mc.json') add_QC_finalization('ITSTPCmatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json') add_QC_finalization('TOFMatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json') diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 31d2ad06a..3cf34a924 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1196,7 +1196,12 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): addQCPerTF(taskName='trdDigitsQC', needs=[TRDDigitask['name']], readerCommand='o2-trd-trap-sim', - configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-digits-task.json') + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json') + + addQCPerTF(taskName='trdTrackingQC', + needs=[TRDTRACKINGtask2['name']], + readerCommand='o2-global-track-cluster-reader --track-types "ITS-TPC-TRD,TPC-TRD" --cluster-types none', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json') ### TOF addQCPerTF(taskName='tofDigitsQC', diff --git a/MC/config/QC/json/trd-digits-task.json b/MC/config/QC/json/trd-standalone-task.json similarity index 66% rename from MC/config/QC/json/trd-digits-task.json rename to MC/config/QC/json/trd-standalone-task.json index ae05bdfd3..8ad3f7523 100644 --- a/MC/config/QC/json/trd-digits-task.json +++ b/MC/config/QC/json/trd-standalone-task.json @@ -3,11 +3,8 @@ "config": { "database": { "implementation": "CCDB", - "host": "ccdb-test.cern.ch:8080", - "username": "not_applicable", - "password": "not_applicable", - "name": "not_applicable" - }, + "host": "ccdb-test.cern.ch:8080" + }, "Activity": { "number": "42", "type": "2", @@ -31,8 +28,7 @@ "className": "o2::quality_control_modules::trd::DigitsTask", "moduleName": "QcTRD", "detectorName": "TRD", - "cycleDurationSeconds": "60", - "maxNumberCycles": "-1", + "cycleDurationSeconds": "60", "dataSource": { "type": "direct", "query": "digits:TRD/DIGITS;tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus" @@ -43,7 +39,18 @@ "pulseheightpeaklower": "1.0", "pulseheightpeakupper": "5.0" } - } + }, + "Tracklets": { + "active": "true", + "className": "o2::quality_control_modules::trd::TrackletsTask", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "tracklets:TRD/TRACKLETS;triggers:TRD/TRKTRGRD;noiseMap:TRD/NOISEMAP/0?lifetime=condition&ccdb-path=TRD/Calib/NoiseMapMCM;chamberStatus:TRD/CHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/HalfChamberStatusQC;fedChamberStatus:TRD/FCHSTATUS/0?lifetime=condition&ccdb-path=TRD/Calib/DCSDPsFedChamberStatus" + } + } }, "dataSamplingPolicies": [] } diff --git a/MC/config/QC/json/trd-tracking-task.json b/MC/config/QC/json/trd-tracking-task.json new file mode 100644 index 000000000..f8093814f --- /dev/null +++ b/MC/config/QC/json/trd-tracking-task.json @@ -0,0 +1,44 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "Tracking": { + "active": "true", + "className": "o2::quality_control_modules::trd::TrackingTask", + "moduleName": "QcTRD", + "detectorName": "TRD", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "trackITSTPCTRD:TRD/MATCH_ITSTPC;trigITSTPCTRD:TRD/TRGREC_ITSTPC;trackTPCTRD:TRD/MATCH_TPC;trigTPCTRD:TRD/TRGREC_TPC" + }, + "taskParameters": { + "detailedQC": "false", + "trackSources": "ITS-TPC-TRD,TPC-TRD" + } + } + }, + "dataSamplingPolicies": [] + } +} From 9943c5b23d125c2947ff892498570fb09060469a Mon Sep 17 00:00:00 2001 From: Diana <70915994+diana0x0f@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:44:36 +0100 Subject: [PATCH 0423/1239] MFT: new MC track QC task (#1468) --- MC/bin/o2dpg_qc_finalization_workflow.py | 1 + MC/bin/o2dpg_sim_workflow.py | 4 ++ MC/config/QC/json/mft-tracks-mc.json | 49 ++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 MC/config/QC/json/mft-tracks-mc.json diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 6908b4956..593f7a280 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -79,6 +79,7 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('mftDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-digits-0.json', MFTDigitsQCneeds) add_QC_finalization('mftClustersQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-clusters.json') add_QC_finalization('mftTracksQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks.json') + add_QC_finalization('mftMCTracksQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks-mc.json') add_QC_finalization('emcRecoQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') add_QC_finalization('emcBCQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json') diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 3cf34a924..b97c94da3 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1180,6 +1180,10 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): needs=[MFTRECOtask['name']], readerCommand='o2-global-track-cluster-reader --track-types MFT --cluster-types MFT', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks.json') + addQCPerTF(taskName='mftMCTracksQC', + needs=[MFTRECOtask['name']], + readerCommand='o2-global-track-cluster-reader --track-types MFT --cluster-types MFT', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/mft-tracks-mc.json') ### TPC # addQCPerTF(taskName='tpcTrackingQC', diff --git a/MC/config/QC/json/mft-tracks-mc.json b/MC/config/QC/json/mft-tracks-mc.json new file mode 100644 index 000000000..1dcd7774d --- /dev/null +++ b/MC/config/QC/json/mft-tracks-mc.json @@ -0,0 +1,49 @@ +{ + "qc" : { + "config" : { + "database" : { + "implementation" : "CCDB", + "host" : "ccdb-test.cern.ch:8080", + "username" : "not_applicable", + "password" : "not_applicable", + "name" : "not_applicable" + }, + "Activity" : { + "number" : "42", + "type" : "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring" : { + "url" : "no-op://" + }, + "consul" : { + "url" : "" + }, + "conditionDB" : { + "url" : "alice-ccdb.cern.ch" + } + }, + "tasks" : { + "TracksMC" : { + "active" : "true", + "className" : "o2::quality_control_modules::mft::QcMFTTrackMCTask", + "moduleName" : "QcMFT", + "detectorName" : "MFT", + "cycleDurationSeconds" : "30", + "maxNumberCycles" : "-1", + "dataSource_comment" : "The other type of dataSource is \"direct\", see basic-no-sampling.json.", + "dataSource" : { + "type" : "direct", + "query" : "tracks:MFT/TRACKS/0;mctruth:MFT/TRACKSMCTR/0" + }, + "location" : "remote", + "taskParameters" : { + "collisionsContextPath": "./collisioncontext.root" + } + } + } + } + } + \ No newline at end of file From e059331566317ae9679d50d5ceb6c6d1a37daa18 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Fri, 8 Mar 2024 15:41:36 +0100 Subject: [PATCH 0424/1239] Adjusting the logic for the rate limiting in split wf --- .../configurations/asyncReco/async_pass.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/DATA/production/configurations/asyncReco/async_pass.sh b/DATA/production/configurations/asyncReco/async_pass.sh index 17d502b01..e5f2e3284 100755 --- a/DATA/production/configurations/asyncReco/async_pass.sh +++ b/DATA/production/configurations/asyncReco/async_pass.sh @@ -586,10 +586,15 @@ else echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 3) matching, calib, AOD, potentially QC" echo -e "\nStep 3) matching, calib, AOD, potentially QC" >> workflowconfig.log - export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-0} + # This uses the same time frame rate limiting as in full wf, unless differently specified in the JDL + export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-${TIMEFRAME_RATE_LIMIT}} echo "Removing detectors $DETECTORS_EXCLUDE" - READER_DELAY=${ALIEN_JDL_READERDELAY:-30} - export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " + if [[ $ALIEN_JDL_USEREADERDELAY == 1 ]]; then + # if we add a delay, the rate limiting should be disabled + TIMEFRAME_RATE_LIMIT=0 + READER_DELAY=${ALIEN_JDL_READERDELAY:-30} + export ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader+=" --reader-delay $READER_DELAY " + fi echo "extra args are $ARGS_EXTRA_PROCESS_o2_global_track_cluster_reader" env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it @@ -620,8 +625,8 @@ else echo "WORKFLOW_PARAMETERS=$WORKFLOW_PARAMETERS" echo "Step 4) QC" echo -e "\nStep 4) QC" >> workflowconfig.log - export TIMEFRAME_RATE_LIMIT=${ALIEN_JDL_TIMEFRAMERATELIMITSSPLITWF:-0} echo "Removing detectors $DETECTORS_EXCLUDE" + echo "The rate limiting will be the same as in step 3: TIMEFRAME_RATE_LIMIT = ${TIMEFRAME_RATE_LIMIT}" env $SETTING_ROOT_OUTPUT IS_SIMULATED_DATA=0 WORKFLOWMODE=print TFDELAY=$TFDELAYSECONDS WORKFLOW_DETECTORS=ALL WORKFLOW_DETECTORS_EXCLUDE=$DETECTORS_EXCLUDE WORKFLOW_DETECTORS_USE_GLOBAL_READER_TRACKS=ALL WORKFLOW_DETECTORS_USE_GLOBAL_READER_CLUSTERS=ALL WORKFLOW_DETECTORS_EXCLUDE_GLOBAL_READER_TRACKS=HMP WORKFLOW_DETECTORS_EXCLUDE_QC=CPV,$DETECTORS_EXCLUDE ./run-workflow-on-inputlist.sh $INPUT_TYPE list.list >> workflowconfig.log # run it if [[ "0$RUN_WORKFLOW" != "00" ]]; then From 1cde53e752a8e9af7a9c8337e1ff497c8a154e74 Mon Sep 17 00:00:00 2001 From: motomioya <95481703+motomioya@users.noreply.github.com> Date: Sat, 9 Mar 2024 00:57:04 +0900 Subject: [PATCH 0425/1239] Add files to simulate bbtomuons without forcing semileptonic decay (#1504) --- .../generator/GeneratorBeautyToMu_EvtGen.C | 5 +- .../GeneratorHF_bbbarToDDbarToMuons_fwdy.ini | 23 +++++ .../GeneratorHF_bbbarToDDbarToMuons_fwdy.C | 85 +++++++++++++++++++ .../PWGDQ/runBeautyToMuons_noForce_fwd_pp.sh | 27 ++++++ 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 MC/config/PWGDQ/ini/GeneratorHF_bbbarToDDbarToMuons_fwdy.ini create mode 100644 MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToDDbarToMuons_fwdy.C create mode 100644 MC/run/PWGDQ/runBeautyToMuons_noForce_fwd_pp.sh diff --git a/MC/config/PWGDQ/external/generator/GeneratorBeautyToMu_EvtGen.C b/MC/config/PWGDQ/external/generator/GeneratorBeautyToMu_EvtGen.C index 98b71a00b..3edfb93dc 100644 --- a/MC/config/PWGDQ/external/generator/GeneratorBeautyToMu_EvtGen.C +++ b/MC/config/PWGDQ/external/generator/GeneratorBeautyToMu_EvtGen.C @@ -11,7 +11,7 @@ R__ADD_INCLUDE_PATH($O2DPG_ROOT/MC/config/PWGHF/external/generator) FairGenerator* -GeneratorBeautyToMu_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = -2.2, bool ispp = true, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") +GeneratorBeautyToMu_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = -2.2, bool ispp = true, bool forcedecay = true, bool verbose = false, TString pdgs = "511;521;531;541;5112;5122;5232;5132;5332") { auto gen = new o2::eventgen::GeneratorEvtGen(); gen->setRapidity(rapidityMin,rapidityMax); @@ -30,7 +30,8 @@ GeneratorBeautyToMu_EvtGenFwdY(double rapidityMin = -4.3, double rapidityMax = - gen->AddPdg(std::stoi(spdg),i); printf("PDG %d \n",std::stoi(spdg)); } - gen->SetForceDecay(kEvtSemiMuonic); + if(forcedecay) gen->SetForceDecay(kEvtSemiMuonic); + else gen->SetForceDecay(kEvtAll); // set random seed gen->readString("Random:setSeed on"); uint random_seed; diff --git a/MC/config/PWGDQ/ini/GeneratorHF_bbbarToDDbarToMuons_fwdy.ini b/MC/config/PWGDQ/ini/GeneratorHF_bbbarToDDbarToMuons_fwdy.ini new file mode 100644 index 000000000..18a7faf43 --- /dev/null +++ b/MC/config/PWGDQ/ini/GeneratorHF_bbbarToDDbarToMuons_fwdy.ini @@ -0,0 +1,23 @@ + +### The setup uses an external event generator +### This part sets the path of the file and the function call to retrieve it + +[GeneratorExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/external/generator/GeneratorBeautyToMu_EvtGen.C +funcName = GeneratorBeautyToMu_EvtGenFwdY(-4.3,-2.3,true,false) + +### The external generator derives from GeneratorPythia8. +### This part configures the bits of the interface: configuration and user hooks + +[GeneratorPythia8] +config = ${O2DPG_ROOT}/MC/config/common/pythia8/generator/pythia8_hf.cfg +hooksFileName = ${O2DPG_ROOT}/MC/config/PWGHF/pythia8/hooks/pythia8_userhooks_qqbar.C +hooksFuncName = pythia8_userhooks_bbbar(-4.3,-2.3) + +### The setup uses an external even generator trigger which is +### defined in the following file and it is retrieved and configured +### according to the specified function call + +[TriggerExternal] +fileName = ${O2DPG_ROOT}/MC/config/PWGDQ/trigger/selectDaughterFromHFwithinAcc.C +funcName = selectDaughterFromHFwithinAcc(13,kTRUE,-4.3,-2.3) diff --git a/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToDDbarToMuons_fwdy.C b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToDDbarToMuons_fwdy.C new file mode 100644 index 000000000..271e0ff39 --- /dev/null +++ b/MC/config/PWGDQ/ini/tests/GeneratorHF_bbbarToDDbarToMuons_fwdy.C @@ -0,0 +1,85 @@ +int External() +{ + int checkPdgDecay = 13; + std::string path{"o2sim_Kine.root"}; + TFile file(path.c_str(), "READ"); + if (file.IsZombie()) { + std::cerr << "Cannot open ROOT file " << path << "\n"; + return 1; + } + auto tree = (TTree*)file.Get("o2sim"); + std::vector* tracks{}; + tree->SetBranchAddress("MCTrack", &tracks); + + int nLeptons{}; + int nLeptonsInAcceptance{}; + int nLeptonsToBeDone{}; + int nSignalPairs{}; + int nLeptonPairs{}; + int nLeptonPairsInAcceptance{}; + int nLeptonPairsToBeDone{}; + auto nEvents = tree->GetEntries(); + + for (int i = 0; i < nEvents; i++) { + tree->GetEntry(i); + int nleptonseinacc = 0; + int nleptonse = 0; + int nleptonseToBeDone = 0; + int nopenHeavy = 0; + for (auto& track : *tracks) { + auto pdg = track.GetPdgCode(); + auto y = track.GetRapidity(); + if (std::abs(pdg) == checkPdgDecay) { + int igmother = track.getMotherTrackId(); + if (igmother > 0) { + auto gmTrack = (*tracks)[igmother]; + int gmpdg = gmTrack.GetPdgCode(); + if ( int(std::abs(gmpdg)/100.) == 4 || int(std::abs(gmpdg)/1000.) == 4 || int(std::abs(gmpdg)/100.) == 5 || int(std::abs(gmpdg)/1000.) == 5 ) { + nLeptons++; + nleptonse++; + if (-4.3 < y && y < -2.3) { + nleptonseinacc++; + nLeptonsInAcceptance++; + } + if (track.getToBeDone()) { + nLeptonsToBeDone++; + nleptonseToBeDone++; + } + } + } + } else if (std::abs(pdg) == 411 || std::abs(pdg) == 421 || std::abs(pdg) == 431 || std::abs(pdg) == 4122 || std::abs(pdg) == 4132 || std::abs(pdg) == 4232 || std::abs(pdg) == 4332 || std::abs(pdg) == 511 || std::abs(pdg) == 521 || std::abs(pdg) == 531 || std::abs(pdg) == 541 || std::abs(pdg) == 5112 || std::abs(pdg) == 5122 || std::abs(pdg) == 5232 || std::abs(pdg) == 5132 || std::abs(pdg) == 5332) { + nopenHeavy++; + } + } + if (nopenHeavy > 1) nSignalPairs++; + if (nleptonse > 1) nLeptonPairs++; + if (nleptonseToBeDone > 1) nLeptonPairsToBeDone++; + if (nleptonseinacc > 1) nLeptonPairsInAcceptance++; + } + std::cout << "#events: " << nEvents << "\n" + << "#muons in acceptance: " << nLeptonsInAcceptance << "\n" + << "#muon pairs in acceptance: " << nLeptonPairsInAcceptance << "\n" + << "#muons: " << nLeptons << "\n" + << "#muons to be done: " << nLeptonsToBeDone << "\n" + << "#signal pairs: " << nSignalPairs << "\n" + << "#muon pairs: " << nLeptonPairs << "\n" + << "#muon pairs to be done: " << nLeptonPairsToBeDone << "\n"; + if (nSignalPairs <= nLeptonPairs) { + std::cerr << "Number of muon pairs should be less than HF hadron pairs\n"; + return 1; + } + if (nSignalPairs < nEvents) { + std::cerr << "Number of signal pairs should be at least equaled to the number of events\n"; + return 1; + } + if (nLeptonPairs != nLeptonPairsToBeDone) { + std::cerr << "The number of muon pairs should be the same as the number of muon pairs which should be transported.\n"; + return 1; + } + if (nLeptons != nLeptonsToBeDone) { + std::cerr << "The number of muons should be the same as the number of muons which should be transported.\n"; + return 1; + } + + return 0; +} diff --git a/MC/run/PWGDQ/runBeautyToMuons_noForce_fwd_pp.sh b/MC/run/PWGDQ/runBeautyToMuons_noForce_fwd_pp.sh new file mode 100644 index 000000000..ded2c646b --- /dev/null +++ b/MC/run/PWGDQ/runBeautyToMuons_noForce_fwd_pp.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# make sure O2DPG + O2 is loaded +[ ! "${O2DPG_ROOT}" ] && echo "Error: This needs O2DPG loaded" && exit 1 +[ ! "${O2_ROOT}" ] && echo "Error: This needs O2 loaded" && exit 1 + +# ----------- SETUP LOCAL CCDB CACHE -------------------------- +export ALICEO2_CCDB_LOCALCACHE=$PWD/.ccdb + + +# ----------- LOAD UTILITY FUNCTIONS -------------------------- +. ${O2_ROOT}/share/scripts/jobutils.sh + +RNDSEED=${RNDSEED:-0} +NSIGEVENTS=${NSIGEVENTS:-1} +NBKGEVENTS=${NBKGEVENTS:-1} +NWORKERS=${NWORKERS:-8} +NTIMEFRAMES=${NTIMEFRAMES:-1} + + +${O2DPG_ROOT}/MC/bin/o2dpg_sim_workflow.py -eCM 13600 -gen external -j ${NWORKERS} -ns ${NSIGEVENTS} -tf ${NTIMEFRAMES} -e TGeant4 -mod "MCH MFT MID ITS" \ + -trigger "external" -ini $O2DPG_ROOT/MC/config/PWGDQ/ini/GeneratorHF_bbbarToDDbarToMuons_fwdy.ini \ + -genBkg pythia8 -procBkg cdiff -colBkg pp --embedding -nb ${NBKGEVENTS} \ + -confKeyBkg "Diamond.width[2]=6" -interactionRate 2000 --mft-assessment-full --fwdmatching-assessment-full + +# run workflow +${O2DPG_ROOT}/MC/bin/o2_dpg_workflow_runner.py -f workflow.json -f workflow.json -tt aod -jmax 1 From 93ba6ee13cded1fd1c20ace47b33e549653d21b2 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Tue, 12 Mar 2024 15:16:04 +0100 Subject: [PATCH 0426/1239] Add first batch of PWG experts as code owners (#1529) --- CODEOWNERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CODEOWNERS b/CODEOWNERS index 27ea257cd..3a3568f07 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,4 +18,10 @@ /DATA/testing/detectors/ZDC /MC @sawenzel @chiarazampolli @gconesab @benedikt-voelkel + +# PWG experts +/MC/*/PWGHF @AliceO2Group/reviewers-pwg-hf @sawenzel @chiarazampolli @benedikt-voelkel +/MC/*/PWGLF @AliceO2Group/reviewers-pwg-lf @sawenzel @chiarazampolli @benedikt-voelkel +/MC/*/PWGEM @AliceO2Group/reviewers-pwg-em @sawenzel @chiarazampolli @benedikt-voelkel + /RelVal @sawenzel @chiarazampolli @gconesab @benedikt-voelkel From fe71ee90d7e37227407f63588b60d106cb9890f4 Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Wed, 13 Mar 2024 13:22:20 +0100 Subject: [PATCH 0427/1239] [RelVal] Fix to really skip unreadable objects (#1530) * e.g. missing dictionary in case of QC files that have been constructed with older SW tags Co-authored-by: Benedikt Volkel --- RelVal/utils/ExtractAndFlatten.C | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/RelVal/utils/ExtractAndFlatten.C b/RelVal/utils/ExtractAndFlatten.C index 5a1ae7dde..889ed0e47 100644 --- a/RelVal/utils/ExtractAndFlatten.C +++ b/RelVal/utils/ExtractAndFlatten.C @@ -160,7 +160,7 @@ void ExtractAndFlattenDirectory(TDirectory* inDir, TDirectory* outDir, std::vect ExtractTree(tree, outDir, collectNames, basedOnTree, currentPrefix); } else { if (!WriteObject(obj, outDir, collectNames, currentPrefix)) { - std::cerr << "Cannot handle object " << obj->GetName() << " which is of class " << key->GetClassName() << "\n"; + std::cerr << "Cannot handle object of class " << key->GetClassName() << "\n"; } } } @@ -261,6 +261,10 @@ void adjustName(TObject* o) // decide which concrete function to call to write the given object bool WriteObject(TObject* o, TDirectory* outDir, std::vector& collectNames, std::string const& currentPrefix) { + if (!o) { + std::cerr << "WARNING: Cannot process object, nullptr received.\n"; + return false; + } if (auto monObj = dynamic_cast(o)) { return WriteObject(monObj->getObject(), outDir, collectNames, currentPrefix); } From cb357d0bc5350f4f83b78ce094b5c4ca1727f794 Mon Sep 17 00:00:00 2001 From: lmassacr Date: Thu, 14 Mar 2024 09:33:00 +0100 Subject: [PATCH 0428/1239] porting of MID aQC workflow to MC (#1531) --- MC/bin/o2dpg_qc_finalization_workflow.py | 2 + MC/bin/o2dpg_sim_workflow.py | 7 ++ MC/config/QC/json/mid-task.json | 124 +++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 MC/config/QC/json/mid-task.json diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 593f7a280..58c26bd44 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -93,6 +93,8 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('TOFMatchWithTRDQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tofMatchedTracks_AllTypes_direct_MC.json') add_QC_finalization('ITSTrackSimTask', 'json://${O2DPG_ROOT}/MC/config/QC/json/its-mc-tracks-qc.json') add_QC_finalization('ITSTracksClusters', 'json://${O2DPG_ROOT}/MC/config/QC/json/its-clusters-tracks-qc.json') + if isActive('MID'): + add_QC_finalization('MIDTaskQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/mid-task.json') if isActive('FT0') and isActive('TRD'): add_QC_finalization('tofft0PIDQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/pidft0tof.json') elif isActive('FT0'): diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index b97c94da3..6fa1ac776 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -1300,6 +1300,13 @@ def addQCPerTF(taskName, needs, readerCommand, configFilePath, objectsFile=''): needs=[PHSRECOtask['name']], readerCommand='o2-phos-reco-workflow --input-type cells --output-type clusters --disable-mc --disable-root-output', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/phs-cells-clusters-task.json') + + ### MID + if isActive('MID'): + addQCPerTF(taskName='MIDTaskQC', + needs=[MIDRECOtask['name']], + readerCommand='o2-mid-digits-reader-workflow | o2-mid-tracks-reader-workflow', + configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/mid-task.json') #secondary vertexer svfinder_threads = ' --threads 1 ' diff --git a/MC/config/QC/json/mid-task.json b/MC/config/QC/json/mid-task.json new file mode 100644 index 000000000..1f76d7ef2 --- /dev/null +++ b/MC/config/QC/json/mid-task.json @@ -0,0 +1,124 @@ +{ + "qc": { + "config": { + "database": { + "implementation": "CCDB", + "host": "ccdb-test.cern.ch:8080", + "username": "not_applicable", + "password": "not_applicable", + "name": "not_applicable" + }, + "Activity": { + "number": "42", + "type": "2", + "provenance": "qc_mc", + "passName": "passMC", + "periodName": "SimChallenge" + }, + "monitoring": { + "url": "no-op://" + }, + "consul": { + "url": "" + }, + "conditionDB": { + "url": "alice-ccdb.cern.ch" + } + }, + "tasks": { + "MIDDigits": { + "active": "true", + "taskName": "Digits", + "className": "o2::quality_control_modules::mid::DigitsQcTask", + "moduleName": "QcMID", + "detectorName": "MID", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "digits:MID/DATA;digits_rof:MID/DATAROF" + } + }, + "MIDClusters": { + "active": "true", + "taskName": "Clusters", + "className": "o2::quality_control_modules::mid::ClustQcTask", + "moduleName": "QcMID", + "detectorName": "MID", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "clusters:MID/TRACKCLUSTERS;clusterrofs:MID/TRCLUSROFS" + } + }, + "MIDTracks": { + "active": "true", + "taskName": "Tracks", + "className": "o2::quality_control_modules::mid::TracksQcTask", + "moduleName": "QcMID", + "detectorName": "MID", + "cycleDurationSeconds": "60", + "dataSource": { + "type": "direct", + "query": "tracks:MID/TRACKS;trackrofs:MID/TRACKROFS" + } + } + }, + "checks": { + "MIDDigits": { + "active": "true", + "checkName": "Digits", + "className": "o2::quality_control_modules::mid::DigitsQcCheck", + "moduleName": "QcMID", + "detectorName": "MID", + "policy": "OnAny", + "checkParameters": { + "MeanMultThreshold": "100.", + "MinMultThreshold": "0.0", + "NbOrbitPerTF": "32.", + "LocalBoardScale": "200.0", + "LocalBoardThreshold": "800.0", + "NbBadLocalBoard": "10.", + "NbEmptyLocalBoard": "117." + }, + "dataSource": [ + { + "type": "Task", + "name": "MIDDigits" + } + ] + }, + "MIDClusters": { + "active": "true", + "checkName": "Clusters", + "className": "o2::quality_control_modules::mid::ClustQcCheck", + "moduleName": "QcMID", + "detectorName": "MID", + "policy": "OnAny", + "dataSource": [ + { + "type": "Task", + "name": "MIDClusters" + } + ] + }, + "MIDTracks": { + "active": "true", + "checkName": "Tracks", + "className": "o2::quality_control_modules::mid::TracksQcCheck", + "moduleName": "QcMID", + "detectorName": "MID", + "policy": "OnAny", + "checkParameters": { + "Ratio44Threshold": "0.1" + }, + "dataSource": [ + { + "type": "Task", + "name": "MIDTracks" + } + ] + } + } + }, + "dataSamplingPolicies": [] +} From cd5b9707b96eecbbb3809fdda561ee660b5bfaa2 Mon Sep 17 00:00:00 2001 From: Bong-Hwi Lim Date: Fri, 15 Mar 2024 10:37:51 +0100 Subject: [PATCH 0429/1239] update Lambda1520 pdg code (#1534) From Pythia 8.303, The PDG code of Lambda(1520) has been updated to 102134 instead of 3124 --- .../PWGLF/pythia8/generator/resonancelist.gun | 6 +-- .../pythia8/generator/resonancelistgun.json | 6 +-- .../PWGLF/pythia8/generator/resonances.cfg | 52 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/MC/config/PWGLF/pythia8/generator/resonancelist.gun b/MC/config/PWGLF/pythia8/generator/resonancelist.gun index daedb88f0..4623a8ad0 100644 --- a/MC/config/PWGLF/pythia8/generator/resonancelist.gun +++ b/MC/config/PWGLF/pythia8/generator/resonancelist.gun @@ -12,9 +12,9 @@ -3114 10 0.2 10 -1.2 1.2 genDecayed 3224 10 0.2 10 -1.2 1.2 genDecayed -3224 10 0.2 10 -1.2 1.2 genDecayed -3124 10 0.2 10 -1.2 1.2 genDecayed --3124 10 0.2 10 -1.2 1.2 genDecayed +102134 10 0.2 10 -1.2 1.2 genDecayed +-102134 10 0.2 10 -1.2 1.2 genDecayed 3324 10 0.2 10 -1.2 1.2 genDecayed -3324 10 0.2 10 -1.2 1.2 genDecayed 10323 10 0.2 10 -1.2 1.2 genDecayed --10323 10 0.2 10 -1.2 1.2 genDecayed \ No newline at end of file +-10323 10 0.2 10 -1.2 1.2 genDecayed diff --git a/MC/config/PWGLF/pythia8/generator/resonancelistgun.json b/MC/config/PWGLF/pythia8/generator/resonancelistgun.json index 5ec4ec892..2d2ae56ef 100644 --- a/MC/config/PWGLF/pythia8/generator/resonancelistgun.json +++ b/MC/config/PWGLF/pythia8/generator/resonancelistgun.json @@ -90,7 +90,7 @@ "genDecayed": true }, "Lambda(1520)0" : { - "pdg": 3124, + "pdg": 102134, "n": 1, "ptMin": 0.0, "ptMax": 20, @@ -99,7 +99,7 @@ "genDecayed": true }, "anti-Lambda(1520)0" : { - "pdg": -3124, + "pdg": -102134, "n": 1, "ptMin": 0.0, "ptMax": 20, @@ -179,4 +179,4 @@ "etaMax": 1.2, "genDecayed": true } -} \ No newline at end of file +} diff --git a/MC/config/PWGLF/pythia8/generator/resonances.cfg b/MC/config/PWGLF/pythia8/generator/resonances.cfg index 6069b5a90..7153511c8 100644 --- a/MC/config/PWGLF/pythia8/generator/resonances.cfg +++ b/MC/config/PWGLF/pythia8/generator/resonances.cfg @@ -2,33 +2,33 @@ ProcessLevel:all = off # will not look for the 'process' # id::all = name antiName spinType chargeType colType m0 mWidth mMin mMax tau0 -3124:all = Lambda1520 Lambda1520bar 0 0 0 1.5195 0.01560 0.06240 0.00000E+00 0 1 +102134:all = Lambda1520 Lambda1520bar 0 0 0 1.5195 0.01560 0.06240 0.00000E+00 0 1 ### add Resonance decays absent in PYTHIA8 decay table and set BRs from PDG for other -3124:oneChannel = 1 0.223547 0 2212 -321 -3124:addChannel = 1 0.223547 0 2112 -311 -3124:addChannel = 1 0.139096 0 3222 -211 -3124:addChannel = 1 0.139096 0 3212 111 -3124:addChannel = 1 0.139096 0 3112 211 -3124:addChannel = 1 0.028780 0 3224 -211 -3124:addChannel = 1 0.028780 0 3214 111 -3124:addChannel = 1 0.028780 0 3114 211 -3124:addChannel = 1 0.019373 0 3212 22 -3124:addChannel = 1 0.014638 0 3122 211 -3124:addChannel = 1 0.007948 0 3122 22 -3124:addChannel = 1 0.007319 0 3122 111 +102134:oneChannel = 1 0.223547 0 2212 -321 +102134:addChannel = 1 0.223547 0 2112 -311 +102134:addChannel = 1 0.139096 0 3222 -211 +102134:addChannel = 1 0.139096 0 3212 111 +102134:addChannel = 1 0.139096 0 3112 211 +102134:addChannel = 1 0.028780 0 3224 -211 +102134:addChannel = 1 0.028780 0 3214 111 +102134:addChannel = 1 0.028780 0 3114 211 +102134:addChannel = 1 0.019373 0 3212 22 +102134:addChannel = 1 0.014638 0 3122 211 +102134:addChannel = 1 0.007948 0 3122 22 +102134:addChannel = 1 0.007319 0 3122 111 -3124:onMode = off +102134:onMode = off -3124:onIfMatch = 2212 -321 -3124:onIfMatch = 2112 -311 -3124:onIfMatch = 3222 -211 -3124:onIfMatch = 3212 111 -3124:onIfMatch = 3112 211 -3124:onIfMatch = 3224 -211 -3124:onIfMatch = 3214 111 -3124:onIfMatch = 3114 211 -3124:onIfMatch = 3212 22 -3124:onIfMatch = 3122 211 -3124:onIfMatch = 3122 22 -3124:onIfMatch = 3122 111 \ No newline at end of file +102134:onIfMatch = 2212 -321 +102134:onIfMatch = 2112 -311 +102134:onIfMatch = 3222 -211 +102134:onIfMatch = 3212 111 +102134:onIfMatch = 3112 211 +102134:onIfMatch = 3224 -211 +102134:onIfMatch = 3214 111 +102134:onIfMatch = 3114 211 +102134:onIfMatch = 3212 22 +102134:onIfMatch = 3122 211 +102134:onIfMatch = 3122 22 +102134:onIfMatch = 3122 111 From 0f090bc760217d7ea379afcfc7f3bf16402c67ee Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 15 Mar 2024 11:41:51 +0100 Subject: [PATCH 0430/1239] PHS: modernize Ped and LED calibration workflows --- DATA/production/calib/phs-led.sh | 83 ++++++++++++++++++-------- DATA/production/calib/phs-pedestal.sh | 85 ++++++++++++++++++--------- 2 files changed, 115 insertions(+), 53 deletions(-) diff --git a/DATA/production/calib/phs-led.sh b/DATA/production/calib/phs-led.sh index b92e78b92..dfd762745 100755 --- a/DATA/production/calib/phs-led.sh +++ b/DATA/production/calib/phs-led.sh @@ -4,36 +4,67 @@ source common/setenv.sh source common/getCommonArgs.sh -if [ -z $PHS_MAX_STATISTICS ]; then - PHS_MAX_STATISTICS=1000000 +source common/gen_topo_helper_functions.sh + +max_statistics=50000 +if [[ ! -z ${PHS_MAX_STATISTICS:-} ]]; then + max_statistics=$PHS_MAX_STATISTICS fi -PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" EXTRA_CONFIG=" " -if [ -z $PHS_CCDB_PATH ]; then - PHS_CCDB_PATH="http://o2-ccdb.internal" +push_ccdb_path="http://o2-ccdb.internal" +pull_ccdb_path="http://o2-ccdb.internal" +if [[ ! -z ${PUHS_CCDB_PATH:-} ]]; then + push_ccdb_path=$PUHS_CCDB_PATH +fi + +if [[ ! -z ${PULL_CCDB_PATH:-} ]]; then + pull_ccdb_path=$PULL_CCDB_PATH +fi + +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + push_ccdb_path="http://ccdb-test.cern.ch:8080" +fi + +QC_CONFIG="/o2/components/qc/ANY/any/phs-led-qc" + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-phos-reco-workflow "--input-type raw --output-type cells --disable-root-input --disable-root-output --keepHGLG on --condition-backend ${pull_ccdb_path}" +add_W o2-phos-calib-workflow "--hglgratio on --statistics ${max_statistics} --forceupdate" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}" +workflow_has_parameter QC && add_QC_from_consul "${QC_CONFIG}" + +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" + +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW fi -QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-led-qc - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" --inject-missing-data \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-phos-reco-workflow $ARGS_ALL \ - --input-type raw \ - --output-type cells \ - --disable-root-input \ - --disable-root-output \ - --keepHGLG on \ - | o2-phos-calib-workflow $ARGS_ALL \ - --hglgratio on \ - --statistics $PHS_MAX_STATISTICS \ - --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \ - --forceupdate \ - | o2-qc $ARGS_ALL \ - --config $QC_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - --ccdb-path $PHS_CCDB_PATH \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} +#o2-dpl-raw-proxy $ARGS_ALL \ +# --dataspec "$PROXY_INSPEC" --inject-missing-data \ +# --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ +# | o2-phos-reco-workflow $ARGS_ALL \ +# --input-type raw \ +# --output-type cells \ +# --disable-root-input \ +# --disable-root-output \ +# --keepHGLG on \ +# | o2-phos-calib-workflow $ARGS_ALL \ +# --hglgratio on \ +# --statistics $PHS_MAX_STATISTICS \ +# --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \ +# --forceupdate \ +# | o2-qc $ARGS_ALL \ +# --config $QC_CONFIG \ +# | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ +# --ccdb-path $PHS_CCDB_PATH \ +# | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} diff --git a/DATA/production/calib/phs-pedestal.sh b/DATA/production/calib/phs-pedestal.sh index 7c285a8f6..a3b2d9d3e 100755 --- a/DATA/production/calib/phs-pedestal.sh +++ b/DATA/production/calib/phs-pedestal.sh @@ -4,36 +4,67 @@ source common/setenv.sh source common/getCommonArgs.sh -if [ -z $PHS_MAX_STATISTICS ] ; then - PHS_MAX_STATISTICS=10000 +source common/gen_topo_helper_functions.sh + +max_statistics=5000 +if [[ ! -z ${PHS_MAX_STATISTICS:-} ]]; then + max_statistics=$PHS_MAX_STATISTICS +fi + +PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0" + + +push_ccdb_path="http://o2-ccdb.internal" +pull_ccdb_path="http://o2-ccdb.internal" +if [[ ! -z ${PUHS_CCDB_PATH:-} ]]; then + push_ccdb_path=$PUHS_CCDB_PATH fi -PROXY_INSPEC="A:PHS/RAWDATA;dd:FLP/DISTSUBTIMEFRAME/0;eos:***/INFORMATION" +if [[ ! -z ${PULL_CCDB_PATH:-} ]]; then + pull_ccdb_path=$PULL_CCDB_PATH +fi + +if [[ $RUNTYPE == "SYNTHETIC" || "${GEN_TOPO_DEPLOYMENT_TYPE:-}" == "ALICE_STAGING" ]]; then + push_ccdb_path="http://ccdb-test.cern.ch:8080" +fi + +QC_CONFIG="/o2/components/qc/ANY/any/phs-pedestal-qc" + +WORKFLOW= +add_W o2-dpl-raw-proxy "--dataspec \"$PROXY_INSPEC\" --inject-missing-data --channel-config \"name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1\"" "" 0 +add_W o2-phos-reco-workflow "--input-type raw --output-type cells --pedestal on --disable-root-input --disable-root-output --condition-backend ${pull_ccdb_path}" +add_W o2-phos-calib-workflow "--pedestals --statistics ${max_statistics} --forceupdate" +#add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}" +workflow_has_parameter QC && add_QC_from_consul "${QC_CONFIG}" +add_W o2-calibration-ccdb-populator-workflow "--ccdb-path ${push_ccdb_path}" -EXTRA_CONFIG=" " +WORKFLOW+="o2-dpl-run ${ARGS_ALL} ${GLOBALDPLOPT}" -if [ -z $PHS_CCDB_PATH ] ; then - PHS_CCDB_PATH="http://o2-ccdb.internal" +if [ $WORKFLOWMODE == "print" ]; then + echo Workflow command: + echo $WORKFLOW | sed "s/| */|\n/g" +else + # Execute the command we have assembled + WORKFLOW+=" --$WORKFLOWMODE ${WORKFLOWMODE_FILE}" + eval $WORKFLOW fi -QC_CONFIG=consul-json://alio2-cr1-hv-con01.cern.ch:8500/o2/components/qc/ANY/any/phs-pedestal-qc - -o2-dpl-raw-proxy $ARGS_ALL \ - --dataspec "$PROXY_INSPEC" --inject-missing-data \ - --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ - | o2-phos-reco-workflow $ARGS_ALL \ - --input-type raw \ - --output-type cells \ - --pedestal on \ - --disable-root-input \ - --disable-root-output \ - | o2-phos-calib-workflow $ARGS_ALL \ - --pedestals \ - --statistics $PHS_MAX_STATISTICS \ - --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \ - --forceupdate \ - | o2-qc $ARGS_ALL \ - --config $QC_CONFIG \ - | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ - --ccdb-path $PHS_CCDB_PATH \ - | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} +#o2-dpl-raw-proxy $ARGS_ALL \ +# --dataspec "$PROXY_INSPEC" --inject-missing-data \ +# --readout-proxy '--channel-config "name=readout-proxy,type=pull,method=connect,address=ipc://@tf-builder-pipe-0,transport=shmem,rateLogging=1"' \ +# | o2-phos-reco-workflow $ARGS_ALL \ +# --input-type raw \ +# --output-type cells \ +# --pedestal on \ +# --disable-root-input \ +# --disable-root-output \ +# | o2-phos-calib-workflow $ARGS_ALL \ +# --pedestals \ +# --statistics $PHS_MAX_STATISTICS \ +# --configKeyValues "NameConf.mCCDBServer=${PHS_CCDB_PATH}" \ +# --forceupdate \ +# | o2-qc $ARGS_ALL \ +# --config $QC_CONFIG \ +# | o2-calibration-ccdb-populator-workflow $ARGS_ALL \ +# --ccdb-path $PHS_CCDB_PATH \ +# | o2-dpl-run $ARGS_ALL --dds ${WORKFLOWMODE_FILE} From 11169493aa50aa1c503b9ed1d96c20ad184d8dc1 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Fri, 15 Mar 2024 13:26:10 +0100 Subject: [PATCH 0431/1239] TPC reco needs CTP digits spotted when running with non-parallel execution in the pipeline runner --- MC/bin/o2dpg_sim_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 6fa1ac776..d1ae5ced4 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -948,7 +948,7 @@ def getDigiTaskName(det): # ----------- # reco # ----------- - tpcreconeeds=[] + tpcreconeeds=[FT0FV0EMCCTPDIGItask['name']] if not args.combine_tpc_clusterization: # We treat TPC clusterization in multiple (sector) steps in order to # stay within the memory limit or to parallelize over sector from outside (not yet supported within cluster algo) From 0aee2a33729ae2784231facb3a0d239728628c7a Mon Sep 17 00:00:00 2001 From: benedikt-voelkel Date: Fri, 15 Mar 2024 16:10:25 +0100 Subject: [PATCH 0432/1239] Get all labels and change table organisation (#1537) Logic is now the following: 1. Fetch all labels (optionally given some regex). 2. Get closed PRs that have at least one of those labels. 3. Split into merged and not merged. 4. Populate a table per label and list merged PRs. (A PR can therefore appear in more than one table.) 5. Write everything to a txt file. Co-authored-by: Benedikt Volkel --- UTILS/o2dpg_make_github_pr_report.py | 118 ++++++++++++++++++++------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/UTILS/o2dpg_make_github_pr_report.py b/UTILS/o2dpg_make_github_pr_report.py index b6a4ac9c9..f2bd24bda 100755 --- a/UTILS/o2dpg_make_github_pr_report.py +++ b/UTILS/o2dpg_make_github_pr_report.py @@ -6,6 +6,7 @@ import sys import argparse import requests +import re def organise_prs(prs): @@ -29,23 +30,21 @@ def organise_prs(prs): merged_at.append(pr['merged_at']) # sort the merged PRs by their merged timestamp - prs_merged = [pr for _, pr in sorted(zip(merged_at, prs))] + prs_merged = [pr for _, pr in sorted(zip(merged_at, prs_merged))] return prs_merged, prs_other -def get_prs(owner, repo, prod_label, pr_state, include_unmerged, per_page=50, start_page=1, pages=1): +def get_prs(owner, repo, request_labels, pr_state, per_page=50, start_page=1, pages=1): """ Get PRs according to some selection """ # GitHub API endpoint for listing closed pull requests with a specific label - merged_token = '&is:merged=true' if not include_unmerged else '' prs_return = [] has_error = False for page in range(start_page, pages + 1): - url = f'https://api.github.com/repos/{owner}/{repo}/pulls?state={pr_state}{merged_token}&page={page}&per_page={per_page}' - print(f'Fetch PRs accrodring to {url}') + url = f'https://api.github.com/repos/{owner}/{repo}/pulls?state={pr_state}&page={page}&per_page={per_page}' # Send GET request to GitHub API response = requests.get(url) @@ -57,13 +56,13 @@ def get_prs(owner, repo, prod_label, pr_state, include_unmerged, per_page=50, st # PRs to return because we filter on a specific label for pr in prs: labels = pr['labels'] - accept = False + take_pr = False for label in labels: - if label['name'] == prod_label: + if label['name'] in request_labels: # only with the correct the label will be accepted - accept = True + take_pr = True break - if not accept: + if not take_pr: continue # we will end up here if accepted, so append prs_return.append(pr) @@ -80,37 +79,97 @@ def get_prs(owner, repo, prod_label, pr_state, include_unmerged, per_page=50, st return organise_prs(prs_return) -def make_report(prs_merged, prs_other, outfile): +def get_labels(owner, repo, regex=None): + """ + Get the labels that match given regex + """ + # the REST API url + url = f'https://api.github.com/repos/{owner}/{repo}/labels' + # Send GET request to GitHub API + response = requests.get(url) + + if response.status_code != 200: + print(f'ERROR: Problem to retrieve labels for owner {owner} and repository {repo}') + return None + + return [label['name'] for label in response.json() if not repo or re.match(regex, label['name'])] + + +def separate_labels_request_accept(labels, accept_suffix=None): + """ + Disentangle labels and