From 51dde1fbca38c431173e52515b61d0d848f5e27a Mon Sep 17 00:00:00 2001
From: fhoelzken <fhoelzken@ep1.rub.de>
Date: Tue, 15 Oct 2024 14:35:50 +0200
Subject: [PATCH] added gradient clipping and adaptive learning rate to Adam
---
MinFunctions/AdamMinimizer.cc | 145 +++++++++++++++++++++-------------
MinFunctions/AdamMinimizer.hh | 4 +-
2 files changed, 92 insertions(+), 57 deletions(-)
diff --git a/MinFunctions/AdamMinimizer.cc b/MinFunctions/AdamMinimizer.cc
index aa093ae0..1c74d061 100644
--- a/MinFunctions/AdamMinimizer.cc
+++ b/MinFunctions/AdamMinimizer.cc
@@ -26,81 +26,114 @@
#include <string>
#include <math.h>
#include <iomanip>
-
-
#include "MinFunctions/AdamMinimizer.hh"
#include "PwaUtils/GlobalEnv.hh"
#include "ErrLogger/ErrLogger.hh"
#include "PwaUtils/GlobalEnv.hh"
#include "ConfigParser/ParserBase.hh"
-AdamMinimizer::AdamMinimizer(std::shared_ptr<AbsFcn<FCNGradientBase>> theAbsFcnPtr, std::shared_ptr<AbsPawianParameters> upar) :
- AbsPawianMinimizer<FCNGradientBase>(theAbsFcnPtr, upar)
- ,_max_iterations(1000)
- ,_s(upar->Params().size(),0.)
- ,_v(upar->Params().size(),0.)
- ,_learning_rate(0.2)
- ,_iterations(0)
- ,_currentPawianParams(std::shared_ptr<AbsPawianParameters>(upar->Clone()))
- ,_bestLH(1.e20)
- ,_bestPawianParams(std::shared_ptr<AbsPawianParameters>(upar->Clone()))
- ,_noItersWoImprovement(0)
-{
+AdamMinimizer::AdamMinimizer(std::shared_ptr<AbsFcn<FCNGradientBase>> theAbsFcnPtr, std::shared_ptr<AbsPawianParameters> upar)
+ : AbsPawianMinimizer<FCNGradientBase>(theAbsFcnPtr, upar),
+ _max_iterations(5000),
+ _s(upar->Params().size(), 0.0),
+ _v(upar->Params().size(), 0.0),
+ _learning_rate(0.001),
+ _initial_lr(0.001),
+ _decay_rate(0.01),
+ _iterations(0),
+ _currentPawianParams(std::shared_ptr<AbsPawianParameters>(upar->Clone())),
+ _bestLH(1.e20),
+ _bestPawianParams(std::shared_ptr<AbsPawianParameters>(upar->Clone())),
+ _noItersWoImprovement(0),
+ _patience(10) {
}
-AdamMinimizer::~AdamMinimizer()
-{
-}
+AdamMinimizer::~AdamMinimizer() {}
+
+void AdamMinimizer::minimize() {
+
+ const double gradient_tolerance = 1e-8;
+
+ if (_iterations == 0) {
+ double currentLH = _absFcn->operator()(_bestPawianParams->Params());
+ if (currentLH < _bestLH) _bestLH = currentLH;
+ std::cout << "Initial best LH: " << _bestLH << "\tnoIters: " << _iterations << "\tnoItersWoImprovement: " << _noItersWoImprovement << std::endl;
+ }
+
+ while (_iterations <= _max_iterations && _noItersWoImprovement < 1000) {
+ if (_noItersWoImprovement > 0 && _noItersWoImprovement >= _patience) {
+ _currentPawianParams = std::shared_ptr<AbsPawianParameters>(_bestPawianParams->Clone());
+ _patience = std::min(_patience * 1.05, 30.0);
+ _noItersWoImprovement = 0;
+ }
+ std::vector<double> derivatives = _absFcn->Gradient(_currentPawianParams->Params());
+
+ updateParameters(_currentPawianParams, derivatives, _s, _v, _iterations);
-void AdamMinimizer::minimize(){
- if(_iterations==0){
- double currentLH=_absFcn->operator()(_bestPawianParams->Params());
- if(currentLH<_bestLH) _bestLH=currentLH;
- InfoMsg << "best LH: " << _bestLH << "\tnoIters: " << _iterations << "\tnoItersWoImprovement: " << _noItersWoImprovement << endmsg;
- }
- while(_iterations <= _max_iterations && _noItersWoImprovement<100){
- if(_noItersWoImprovement>0 && _noItersWoImprovement%10 == 0){ //continue with parameters of the best fit
- _currentPawianParams=std::shared_ptr<AbsPawianParameters>(_bestPawianParams->Clone());
- }
- std::vector<double> derivatives = _absFcn->Gradient(_currentPawianParams->Params());
-
- updateParameters(_currentPawianParams, derivatives, _s, _v, _iterations);
- double currentLH=_absFcn->operator()(_currentPawianParams->Params());
- if(currentLH<_bestLH){
- _bestLH=currentLH;
- _bestPawianParams=std::shared_ptr<AbsPawianParameters>(_currentPawianParams->Clone());
- _noItersWoImprovement=0;
- }
- else ++_noItersWoImprovement;
- ++_iterations;
- _learning_rate *= 0.999;
- InfoMsg << "best LH: " << _bestLH << "\tnoIters: " << _iterations << "\tnoItersWoImprovement: " << _noItersWoImprovement << endmsg;
+ double currentLH = _absFcn->operator()(_currentPawianParams->Params());
+
+ if (currentLH < _bestLH) {
+ _bestLH = currentLH;
+ _bestPawianParams = std::shared_ptr<AbsPawianParameters>(_currentPawianParams->Clone());
+ _noItersWoImprovement = 0;
+ } else {
+ ++_noItersWoImprovement;
+ }
+
+ double gradientNorm = 0.0;
+ for (const auto& grad : derivatives) {
+ gradientNorm += grad * grad;
+ }
+ gradientNorm = sqrt(gradientNorm);
+
+ if (gradientNorm < gradient_tolerance && std::abs(currentLH - _bestLH) / std::abs(_bestLH) < 1e-4) {
+ std::cout << "Stopping early: Gradient norm below tolerance and no significant likelihood improvement." << std::endl;
+ break;
+ }
+
+
+ ++_iterations;
+ _learning_rate *= 0.999;
+ std::cout << "best LH: " << _bestLH << "\tnoIters: " << _iterations << "\tnoItersWoImprovement: " << _noItersWoImprovement << std::endl;
}
}
-void AdamMinimizer::updateParameters(std::shared_ptr<AbsPawianParameters> pawianParams, std::vector<double>& gradients, std::vector<double>& s, std::vector<double>& v, int t){
- double beta1=0.9;
- double beta2=0.99;
- double epsilon=1.*pow(10., -8);
+void AdamMinimizer::updateParameters(std::shared_ptr<AbsPawianParameters> pawianParams, std::vector<double>& gradients, std::vector<double>& s, std::vector<double>& v, int t) {
+ double beta1 = 0.9;
+ double beta2 = 0.999;
+ double epsilon = 1e-8;
- for(unsigned int i = 0; i < pawianParams->Params().size(); ++i){
- if (pawianParams->IsFixed(i)) continue;
- s.at(i) = beta1 * s.at(i) + (1.0 - beta1) * gradients.at(i);
- v.at(i) = beta2 * v.at(i) + (1.0 - beta2) * gradients.at(i) * gradients.at(i);
+ for (unsigned int i = 0; i < pawianParams->Params().size(); ++i) {
+ if (pawianParams->IsFixed(i)) continue;
+
+ double clip_value = 0.2 * std::abs(pawianParams->Value(i));
+ gradients[i] = std::min(clip_value, std::max(-clip_value, gradients[i]));
- double s_hat = s.at(i) / (1.0 - pow(beta1 , (t + 1)));
- double v_hat = v.at(i) / (1.0 - pow(beta2 , (t + 1)));
+ s.at(i) = beta1 * s.at(i) + (1.0 - beta1) * gradients.at(i);
+ v.at(i) = beta2 * v.at(i) + (1.0 - beta2) * gradients.at(i) * gradients.at(i);
- double newVal = pawianParams->Value(i) - _learning_rate * s_hat / (std::sqrt(v_hat) + epsilon);
- if(pawianParams->HasLimits(i)){
- if(newVal>pawianParams->UpperLimit(i)) newVal=pawianParams->UpperLimit(i);
- else if(newVal<pawianParams->LowerLimit(i)) newVal=pawianParams->LowerLimit(i);
+
+ double s_hat = s.at(i) / (1.0 - pow(beta1, (t + 1)));
+ double v_hat = v.at(i) / (1.0 - pow(beta2, (t + 1)));
+
+
+ //double learning_rate_t = _initial_lr / (1.0 + _decay_rate * t);
+ //double adaptive_lr = learning_rate_t / (std::abs(pawianParams->Value(i)) + epsilon);
+ //double gradient_update = adaptive_lr * s_hat / (std::sqrt(v_hat) + epsilon);
+ double newVal = pawianParams->Value(i) - _learning_rate * s_hat / (std::sqrt(v_hat) + epsilon);
+
+
+ if (pawianParams->HasLimits(i)) {
+ if (newVal > pawianParams->UpperLimit(i)) newVal = pawianParams->UpperLimit(i);
+ else if (newVal < pawianParams->LowerLimit(i)) newVal = pawianParams->LowerLimit(i);
+ }
+
+ pawianParams->SetValue(i, newVal);
}
- pawianParams->SetValue(i,newVal);
- }
}
+
void AdamMinimizer::printFitResult(double evtWeightSumData){
InfoMsg << "\n\n********************** Final fit parameters *************************" << endmsg;
_bestPawianParams->print(std::cout, true);
diff --git a/MinFunctions/AdamMinimizer.hh b/MinFunctions/AdamMinimizer.hh
index ea72023f..3717cf49 100644
--- a/MinFunctions/AdamMinimizer.hh
+++ b/MinFunctions/AdamMinimizer.hh
@@ -61,12 +61,14 @@ protected:
std::vector<double> _s;
std::vector<double> _v;
double _learning_rate;
+ double _initial_lr;
+ double _decay_rate;
unsigned int _iterations;
std::shared_ptr<AbsPawianParameters> _currentPawianParams;
-
double _bestLH;
std::shared_ptr<AbsPawianParameters> _bestPawianParams;
unsigned int _noItersWoImprovement;
+ double _patience;
void updateParameters(std::shared_ptr<AbsPawianParameters> pawianParams, std::vector<double>& gradients, std::vector<double>& s, std::vector<double>& v, int t);
--
GitLab