From b5f316237ed08ab7d8c038fc0c918ed84ee6d4f3 Mon Sep 17 00:00:00 2001
From: Philippe Canal <pcanal@fnal.gov>
Date: Fri, 5 Feb 2021 14:09:16 -0600
Subject: [PATCH] TFileMerger allow to delay the object writes.
Limited to TTree and Histogram.
---
io/io/inc/TFileMerger.h | 7 ++++---
io/io/src/TBufferMerger.cxx | 6 +++++-
io/io/src/TFileMerger.cxx | 25 +++++++++++++++++++++++--
3 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/io/io/inc/TFileMerger.h b/io/io/inc/TFileMerger.h
index 339881a42d1..a7b04ee9a94 100644
--- a/io/io/inc/TFileMerger.h
+++ b/io/io/inc/TFileMerger.h
@@ -66,13 +66,14 @@ public:
kIncremental = BIT(1), ///< Merge the input file with the content of the output file (if already exising).
kResetable = BIT(2), ///< Only the objects with a MergeAfterReset member function.
kNonResetable = BIT(3), ///< Only the objects without a MergeAfterReset member function.
+ kDelayWrite = BIT(4), ///< Delay the TFile write (to reduce the number of write when reusing the file)
kAll = BIT(2)|BIT(3), ///< Merge all type of objects (default)
kAllIncremental = kIncremental | kAll, ///< Merge incrementally all type of objects.
- kOnlyListed = BIT(4), ///< Only the objects specified in fObjectNames list
- kSkipListed = BIT(5), ///< Skip objects specified in fObjectNames list
- kKeepCompression= BIT(6) ///< Keep compression level unchanged for each input files
+ kOnlyListed = BIT(5), ///< Only the objects specified in fObjectNames list
+ kSkipListed = BIT(6), ///< Skip objects specified in fObjectNames list
+ kKeepCompression= BIT(7) ///< Keep compression level unchanged for each input files
};
TFileMerger(Bool_t isLocal = kTRUE, Bool_t histoOneGo = kTRUE);
diff --git a/io/io/src/TBufferMerger.cxx b/io/io/src/TBufferMerger.cxx
index c09ceae192e..6eae4df2dd5 100644
--- a/io/io/src/TBufferMerger.cxx
+++ b/io/io/src/TBufferMerger.cxx
@@ -49,6 +49,10 @@ TBufferMerger::~TBufferMerger()
if (!fQueue.empty())
Merge();
+
+ TFile *out = fMerger.GetOutputFile();
+ if (out)
+ out->Write("",TObject::kOverwrite);
}
std::shared_ptr<TBufferMergerFile> TBufferMerger::GetFile()
@@ -121,7 +125,7 @@ void TBufferMerger::Merge()
queue.pop();
}
- fMerger.PartialMerge();
+ fMerger.PartialMerge(TFileMerger::kAll | TFileMerger::kIncremental | TFileMerger::kDelayWrite);
fMerger.Reset();
fMergeMutex.unlock();
}
diff --git a/io/io/src/TFileMerger.cxx b/io/io/src/TFileMerger.cxx
index ef201431a07..cf8e80d4f56 100644
--- a/io/io/src/TFileMerger.cxx
+++ b/io/io/src/TFileMerger.cxx
@@ -512,6 +512,13 @@ Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t
key->GetName(), key->GetTitle());
continue;
}
+ Bool_t canBeFound = (type & kIncremental) && (current_sourcedir->GetList()->FindObject(key->GetName()) != nullptr);
+ Bool_t needWriteAnyWay = kFALSE;
+ if (canBeFound) {
+ // For now the code require a key (that might be ignored see search through current_sourcedir->GetList())
+ // in the output file in order for it to be even considered.
+ needWriteAnyWay = target->GetListOfKeys()->FindObject(key->GetName()) == nullptr;
+ }
// if (cl->IsTObject())
// obj->ResetBit(kMustCleanup);
if (cl->IsTObject() && cl != obj->IsA()) {
@@ -790,7 +797,7 @@ Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t
}
((TCollection*)obj)->SetOwner();
delete obj;
- } else {
+ } else if (!canBeFound) { // Don't write the partial result for TTree and TH1
// Don't overwrite, if the object were not merged.
// NOTE: this is probably wrong for emulated objects.
if (cl->IsTObject()) {
@@ -804,6 +811,17 @@ Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t
}
}
cl->Destructor(obj); // just in case the class is not loaded.
+ } else if (needWriteAnyWay) {
+ if (cl->IsTObject()) {
+ if ( obj->Write( oldkeyname, canBeMerged ? TObject::kOverwrite : 0) <= 0) {
+ status = kFALSE;
+ }
+ obj->ResetBit(kMustCleanup);
+ } else {
+ if ( target->WriteObjectAny( (void*)obj, cl, oldkeyname, canBeMerged ? "OverWrite" : "" ) <= 0) {
+ status = kFALSE;
+ }
+ }
}
info.Reset();
} // while ( ( TKey *key = (TKey*)nextkey() ) )
@@ -921,8 +939,11 @@ Bool_t TFileMerger::PartialMerge(Int_t in_type)
} else {
// Close or write is required so the file is complete.
if (in_type & kIncremental) {
- fOutputFile->Write("",TObject::kOverwrite);
+ if (!(in_type & kDelayWrite))
+ fOutputFile->Write("",TObject::kOverwrite);
} else {
+ if (type & kIncremental)
+ fOutputFile->Write("",TObject::kOverwrite);
gROOT->GetListOfFiles()->Remove(fOutputFile);
fOutputFile->Close();
}
--
GitLab