From d519905bf9ac9880a9f733b8c309ef0daf00dd80 Mon Sep 17 00:00:00 2001
From: Philippe Canal <pcanal@fnal.gov>
Date: Fri, 12 Feb 2021 14:44:05 -0600
Subject: [PATCH] Skip compression of the top TKeys in the TMemFile by default.
To save space, the compression can be re-enabled by TBuffferMerger::SetCompressTemporaryKeys.
The TBaskets are still compressed and thus the end result is unchanged.
---
io/io/inc/ROOT/TBufferMerger.hxx | 25 +++++++++++++++++++++++++
io/io/src/TBufferMerger.cxx | 2 +-
io/io/src/TBufferMergerFile.cxx | 4 ++++
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/io/io/inc/ROOT/TBufferMerger.hxx b/io/io/inc/ROOT/TBufferMerger.hxx
index 9440d25d9ca..6cc2ff9c3cc 100644
--- a/io/io/inc/ROOT/TBufferMerger.hxx
+++ b/io/io/inc/ROOT/TBufferMerger.hxx
@@ -110,6 +110,30 @@ public:
return fMerger.GetNotrees();
}
+ /** Indicates that the temporary keys (corresponding to the object held by the directories
+ * of the TMemFile) should be compressed or not. Those object are stored in the TMemFile
+ * (and thus possibly compressed) when a thread push its data forward (by calling
+ * TBufferMergerFile::Write) and the queue is being processed by another.
+ * Once the TMemFile is picked (by any thread to be merged), *after* taking the
+ * TBufferMerger::fMergeMutex, those object are read back (and thus possibly uncompressed)
+ * and then used by merging.
+ * In order word, the compression of those objects/keys is only usefull to reduce the size
+ * in memory (of the TMemFile) and does not affect (at all) the compression factor of the end
+ * result.
+ */
+ void SetCompressTemporaryKeys(Bool_t request_compression = true)
+ {
+ fCompressTemporaryKeys = request_compression;
+ }
+
+ /** Returns whether to compressed the TKey in the TMemFile for the object held by
+ * the TDirectories. See TBufferMerger::SetCompressTemporaryKeys for more details.
+ */
+ Bool_t GetCompressTemporaryKeys() const
+ {
+ return fCompressTemporaryKeys;
+ }
+
friend class TBufferMergerFile;
private:
@@ -130,6 +154,7 @@ private:
void Push(TBufferFile *buffer);
bool TryMerge(TBufferMergerFile *memfile);
+ bool fCompressTemporaryKeys{false}; //< Enable compression of the TKeys in the TMemFile (save memory at the expense of time, end result is unchanged)
size_t fAutoSave{0}; //< AutoSave only every fAutoSave bytes
std::atomic<size_t> fBuffered{0}; //< Number of bytes currently buffered
TFileMerger fMerger{false, false}; //< TFileMerger used to merge all buffers
diff --git a/io/io/src/TBufferMerger.cxx b/io/io/src/TBufferMerger.cxx
index 1c083a48d24..e060edfbeb9 100644
--- a/io/io/src/TBufferMerger.cxx
+++ b/io/io/src/TBufferMerger.cxx
@@ -131,7 +131,7 @@ void TBufferMerger::MergeImpl()
queue.pop();
}
- fMerger.PartialMerge(TFileMerger::kAll | TFileMerger::kIncremental | TFileMerger::kDelayWrite);
+ fMerger.PartialMerge(TFileMerger::kAll | TFileMerger::kIncremental | TFileMerger::kDelayWrite | TFileMerger::kKeepCompression);
fMerger.Reset();
}
diff --git a/io/io/src/TBufferMergerFile.cxx b/io/io/src/TBufferMergerFile.cxx
index 9c09d081b11..ce45ba1fe1e 100644
--- a/io/io/src/TBufferMergerFile.cxx
+++ b/io/io/src/TBufferMergerFile.cxx
@@ -42,7 +42,11 @@ Int_t TBufferMergerFile::Write(const char *name, Int_t opt, Int_t bufsize)
return 0;
}
+ auto oldCompLevel = GetCompressionLevel();
+ if (!fMerger.GetCompressTemporaryKeys())
+ SetCompressionLevel(0);
Int_t nbytes = TMemFile::Write(name, opt, bufsize);
+ SetCompressionLevel(oldCompLevel);
if (nbytes) {
TBufferFile *buffer = new TBufferFile(TBuffer::kWrite, GetSize());
--
GitLab