Row 4334

Row ID: 4334 | Dataset Entry | Axioma AXP Content Repository

Content Data

This page contains data entry 4334 from the Axioma AXP content repository. The structured data below represents the complete record for this entry.

Adam optimizer has an adaptive learning rate per parameter that is changing during training. I'm trying to get the average learning rate over all parameters. I found [this SO](https://stackoverflow.com/questions/61773139/how-to-see-the-adapted-learning-rate-for-adam-in-pytorch) question that has a related question, and one of the answers (no accepted answers to this SO question) suggested using

def get_current_lr(optimizer, group_idx, parameter_idx): # Adam has different learning rates for each paramter. So we need to pick the # group and paramter first. group = optimizer.param_groups[group_idx] p = group['params'][parameter_idx] beta1, _ = group['betas'] state = optimizer.state[p] bias_correction1 = 1 - beta1 ** state['step'] current_lr = group['lr'] / bias_correction1 / torch.sqrt(state['exp_avg_sq'] + 1e-8) return current_lr

I tried to adapt it to my case to get the average, but the results don't make much sense, as the learning rate seems to be ridiculously high (sometimes over 15). So I'm wondering if this is the right approach, or am I missing something.

import torch import torch.nn as nn import torch.optim as optim torch.manual_seed(42) def get_current_lr(optimizer): # Adam has different learning rates for each paramter. So we need to pick the # group and paramter first. lrs = [] for group_idx in range(len(optimizer.param_groups)): group = optimizer.param_groups[group_idx] for parameter_idx in range(len(opt.param_groups[group_idx]['params'])): p = group['params'][parameter_idx] beta1, _ = group['betas'] state = optimizer.state[p] bias_correction1 = 1 - beta1 ** state['step'] current_lr = group['lr'] / bias_correction1 / torch.sqrt(state['exp_avg_sq'] + 1e-8) # print(current_lr.mean()) lrs.append(current_lr.mean().item()) return sum(lrs)/len(lrs) class Model(nn.Module): def __init__(self): super(Model, self).__init__() self.fc1 = nn.Linear(10, 100) self.fc2 = nn.Linear(100, 1) def forward(self, x): x = torch.relu(self.fc1(x)) x = self.fc2(x) return x net = Model() # opt = optim.SGD(net.parameters(), lr=1e-2) opt = optim.Adam(net.parameters()) features = torch.rand((100,10)) x_goal = torch.tensor(100) for epoch in range(100): x = net(features) loss = torch.square(x_goal - x).mean() opt.zero_grad() loss.backward() opt.step() if epoch % 5 == 0 : print(get_current_lr(opt)) >>> 16.065366545983125 3.296213309022278 2.23316600310136 1.8703228593794847 1.7041209160006474 1.6200325103309297 1.5739126955249958 1.5481085549622549 1.5332565682870154 1.5246047580963022 1.5195341832059057 1.5165529197865908 1.5148021120267003 1.5137746352747854 1.5131711492076647 1.512817604085285 1.5126127881085267 1.5124981075282449 1.5124379168978521 1.5124109954704181

​

FieldValue
text Adam optimizer has an adaptive learning rate per parameter that is changing during training. I'm trying to get the average learning rate over all parameters. I found [this SO](https://stackoverflow.com/questions/61773139/how-to-see-the-adapted-learning-rate-for-adam-in-pytorch) question that has a related question, and one of the answers (no accepted answers to this SO question) suggested using def get_current_lr(optimizer, group_idx, parameter_idx): # Adam has different learning ra…
label r/pytorch
dataType post
communityName r/pytorch
datetime 2024-04-08
username_encoded Z0FBQUFBQm5LakwxM3NmRURNYjFtUlJfQnZUdUYteXE2WmZqV2FvUVRTUEVOd3dpbi1yX3hYTzJSSXVqNGhZamI4TlpMV1dWbmJHQXJTZWpmQmEzMU0zNlFRWmJDVjQ0YlE9PQ==
url_encoded Z0FBQUFBQm5Lak9Ga0EzOThiUnF3ZVo2QmdTSUZyOWZxMXp2dW1oMS1KOU44WTFaMjJWQmVkQ2FETjVJckdSMF9XNWRIRXdmaWxZSnB0LTV5UFRWc19zSFp0UXdhT2twTGc3cU8wQ1kwNHk4bzdRa0ExaE1Xdnlac0FqejBLWk1wM3lCSTBZNThxRzRFaEtpbWdvTzZ6QmtKVzJUelhreGZGT29WTC1zSXFYOVZ4cllzaXVwMmRxNG5HQ1ZieFVfYktpbmlQQ181V0FFQU1leVgxbzNnQXRqeUdSM2d5dFRvdz09

Raw Record

{
  "text": "Adam optimizer has an adaptive learning rate per parameter that is changing during training. I'm trying to get the average learning rate over all parameters. I found [this SO](https://stackoverflow.com/questions/61773139/how-to-see-the-adapted-learning-rate-for-adam-in-pytorch) question that has a related question, and one of the answers (no accepted answers to this SO question) suggested using\n\n    def get_current_lr(optimizer, group_idx, parameter_idx):\n        # Adam has different learning rates for each paramter. So we need to pick the\n        # group and paramter first.\n        group = optimizer.param_groups[group_idx]\n        p = group['params'][parameter_idx]\n    \n        beta1, _ = group['betas']\n        state = optimizer.state[p]\n    \n        bias_correction1 = 1 - beta1 ** state['step']\n        current_lr = group['lr'] / bias_correction1 / torch.sqrt(state['exp_avg_sq'] + 1e-8)\n        return current_lr\n\nI tried to adapt it to my case to get the average, but the results don't make much sense, as the learning rate seems to be ridiculously high (sometimes over 15). So I'm wondering if this is the right approach, or am I missing something.\n\n    import torch\n    import torch.nn as nn\n    import torch.optim as optim\n    \n    torch.manual_seed(42)\n    \n    def get_current_lr(optimizer):\n        # Adam has different learning rates for each paramter. So we need to pick the\n        # group and paramter first.\n        lrs = []\n        for group_idx in range(len(optimizer.param_groups)):\n            group = optimizer.param_groups[group_idx]\n            for parameter_idx in range(len(opt.param_groups[group_idx]['params'])):\n                p = group['params'][parameter_idx]\n    \n                beta1, _ = group['betas']\n                state = optimizer.state[p]\n    \n                bias_correction1 = 1 - beta1 ** state['step']\n                current_lr = group['lr'] / bias_correction1 / torch.sqrt(state['exp_avg_sq'] + 1e-8)\n                # print(current_lr.mean())\n                lrs.append(current_lr.mean().item())\n    \n        return sum(lrs)/len(lrs)\n    \n    class Model(nn.Module):\n        def __init__(self):\n            super(Model, self).__init__()\n            self.fc1 = nn.Linear(10, 100)\n            self.fc2 = nn.Linear(100, 1)\n    \n        def forward(self, x):\n            x = torch.relu(self.fc1(x))\n            x = self.fc2(x)\n            return x\n    \n    net = Model()\n    # opt = optim.SGD(net.parameters(), lr=1e-2)\n    opt = optim.Adam(net.parameters())\n    \n    features = torch.rand((100,10))\n    x_goal = torch.tensor(100)\n    \n    for epoch in range(100):\n        x = net(features)\n        loss = torch.square(x_goal - x).mean()\n        opt.zero_grad()\n        loss.backward()\n        opt.step()\n        if epoch % 5 == 0 :\n            print(get_current_lr(opt))\n    >>>\n    16.065366545983125\n    3.296213309022278\n    2.23316600310136\n    1.8703228593794847\n    1.7041209160006474\n    1.6200325103309297\n    1.5739126955249958\n    1.5481085549622549\n    1.5332565682870154\n    1.5246047580963022\n    1.5195341832059057\n    1.5165529197865908\n    1.5148021120267003\n    1.5137746352747854\n    1.5131711492076647\n    1.512817604085285\n    1.5126127881085267\n    1.5124981075282449\n    1.5124379168978521\n    1.5124109954704181\n    \n\n​",
  "label": "r/pytorch",
  "dataType": "post",
  "communityName": "r/pytorch",
  "datetime": "2024-04-08",
  "username_encoded": "Z0FBQUFBQm5LakwxM3NmRURNYjFtUlJfQnZUdUYteXE2WmZqV2FvUVRTUEVOd3dpbi1yX3hYTzJSSXVqNGhZamI4TlpMV1dWbmJHQXJTZWpmQmEzMU0zNlFRWmJDVjQ0YlE9PQ==",
  "url_encoded": "Z0FBQUFBQm5Lak9Ga0EzOThiUnF3ZVo2QmdTSUZyOWZxMXp2dW1oMS1KOU44WTFaMjJWQmVkQ2FETjVJckdSMF9XNWRIRXdmaWxZSnB0LTV5UFRWc19zSFp0UXdhT2twTGc3cU8wQ1kwNHk4bzdRa0ExaE1Xdnlac0FqejBLWk1wM3lCSTBZNThxRzRFaEtpbWdvTzZ6QmtKVzJUelhreGZGT29WTC1zSXFYOVZ4cllzaXVwMmRxNG5HQ1ZieFVfYktpbmlQQ181V0FFQU1leVgxbzNnQXRqeUdSM2d5dFRvdz09"
}
Explore Dataset Explore Row

Entry Information