深度学习优化器进化史：从SGD到AdamW的原理与选择

点击 “AladdinEdu，同学们用得起的【H卡】算力平台”，注册即送-H卡级别算力，80G大显存，按量计费，灵活弹性，顶级配置，学生更享专属优惠。

引言：优化器——深度学习的引擎

在深度学习的宏伟殿堂中，优化器扮演着至关重要的角色，它如同模型的引擎，决定了学习过程的效率和最终性能。从最基础的随机梯度下降（SGD）到如今广泛使用的Adam及其变体，优化器的发展历程见证了深度学习领域的飞速进步。

优化器的核心任务是调整模型参数以最小化损失函数，这个过程看似简单，实则充满挑战：如何平衡收敛速度和稳定性？如何处理不同参数的不同学习需求？如何避免陷入局部最优或鞍点？这些问题推动着优化器技术的不断演进。

本文将深入探讨优化器的发展历程，从经典的SGD到现代的自适应优化器，通过理论分析、代码实现和实验对比，帮助读者全面理解各种优化器的原理和特点，并掌握根据具体任务选择合适优化器的能力。

第一部分：优化器基础与数学原理

1.1 优化问题基础

深度学习中的优化问题可以形式化为：

min_θ L(θ) = 𝔼_(x,y)∼𝒟[ℓ(f(x;θ), y)]

其中θ是模型参数，L是损失函数，𝒟是数据分布，ℓ是损失函数，f是模型函数。

1.2 梯度下降的核心思想

梯度下降法基于一个简单而强大的思想：沿着梯度反方向更新参数，因为梯度方向是函数增长最快的方向。

批量梯度下降更新公式：
θ_t+1 = θ_t - η∇L(θ_t)

其中η是学习率。

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from IPython.display import HTML# 定义测试函数
def quadratic_function(x, y):"""二次函数，用于优化演示"""return x**2 + 2*y**2 + x*y + 3*x - 2*y + 5def quadratic_gradient(x, y):"""二次函数的梯度"""dx = 2*x + y + 3dy = 4*y + x - 2return np.array([dx, dy])# 梯度下降可视化
def visualize_gradient_descent():# 创建网格数据x = np.linspace(-5, 5, 100)y = np.linspace(-5, 5, 100)X, Y = np.meshgrid(x, y)Z = quadratic_function(X, Y)# 初始参数params = np.array([-4.0, 4.0])learning_rate = 0.1trajectory = [params.copy()]# 执行梯度下降for i in range(50):grad = quadratic_gradient(params[0], params[1])params = params - learning_rate * gradtrajectory.append(params.copy())trajectory = np.array(trajectory)# 绘制等高线图plt.figure(figsize=(10, 8))contour = plt.contour(X, Y, Z, 50, cmap='viridis')plt.colorbar(contour)plt.plot(trajectory[:, 0], trajectory[:, 1], 'ro-', markersize=4)plt.title('Gradient Descent Optimization Path')plt.xlabel('x')plt.ylabel('y')plt.show()return trajectory# 运行可视化
trajectory = visualize_gradient_descent()

第二部分：经典优化器详解

2.1 随机梯度下降（SGD）

SGD是深度学习中最基础的优化器，它使用单个样本或小批量的梯度来更新参数。

SGD更新公式：
θ_t+1 = θ_t - η∇L(θ_t; x⁽ⁱ⁾, y⁽ⁱ⁾)

import torch
import torch.nn as nn
import torch.optim as optimclass SGDOptimizer:def __init__(self, params, lr=0.01):"""手动实现SGD优化器Args:params: 模型参数lr: 学习率"""self.params = list(params)self.lr = lrdef zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""for param in self.params:if param.grad is not None:param.data = param.data - self.lr * param.grad.data# 使用示例
def test_sgd_optimizer():# 创建简单模型model = nn.Linear(10, 1)# 使用自定义SGD优化器optimizer = SGDOptimizer(model.parameters(), lr=0.01)# 模拟训练步骤for epoch in range(10):# 模拟输入和目标inputs = torch.randn(32, 10)targets = torch.randn(32, 1)# 前向传播outputs = model(inputs)loss = nn.MSELoss()(outputs, targets)# 反向传播optimizer.zero_grad()loss.backward()# 参数更新optimizer.step()print(f'Epoch {epoch+1}, Loss: {loss.item():.4f}')# 测试SGD优化器
test_sgd_optimizer()

2.2 带动量的SGD（Momentum SGD）

动量法通过引入速度变量来加速SGD，并减少振荡。

动量SGD更新公式：
v_t+1 = γv_t + η∇L(θ_t)
θ_t+1 = θ_t - v_t+1

class MomentumSGD:def __init__(self, params, lr=0.01, momentum=0.9):"""手动实现带动量的SGD优化器Args:params: 模型参数lr: 学习率momentum: 动量系数"""self.params = list(params)self.lr = lrself.momentum = momentumself.velocities = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""for i, param in enumerate(self.params):if param.grad is not None:# 更新速度self.velocities[i] = self.momentum * self.velocities[i] + self.lr * param.grad.data# 更新参数param.data = param.data - self.velocities[i]# 动量SGD效果对比
def compare_sgd_vs_momentum():"""对比SGD和动量SGD的优化路径"""# 定义测试函数def himmelblau(x, y):return (x**2 + y - 11)**2 + (x + y**2 - 7)**2def himmelblau_gradient(x, y):dx = 4*x*(x**2 + y - 11) + 2*(x + y**2 - 7)dy = 2*(x**2 + y - 11) + 4*y*(x + y**2 - 7)return np.array([dx, dy])# 初始化参数x_sgd, y_sgd = 0.0, 0.0x_momentum, y_momentum = 0.0, 0.0lr = 0.01momentum = 0.9# 记录轨迹sgd_trajectory = [(x_sgd, y_sgd)]momentum_trajectory = [(x_momentum, y_momentum)]# 速度初始化vx, vy = 0.0, 0.0# 优化过程for i in range(1000):# SGDgrad_sgd = himmelblau_gradient(x_sgd, y_sgd)x_sgd = x_sgd - lr * grad_sgd[0]y_sgd = y_sgd - lr * grad_sgd[1]sgd_trajectory.append((x_sgd, y_sgd))# Momentum SGDgrad_momentum = himmelblau_gradient(x_momentum, y_momentum)vx = momentum * vx + lr * grad_momentum[0]vy = momentum * vy + lr * grad_momentum[1]x_momentum = x_momentum - vxy_momentum = y_momentum - vymomentum_trajectory.append((x_momentum, y_momentum))# 可视化x = np.linspace(-5, 5, 100)y = np.linspace(-5, 5, 100)X, Y = np.meshgrid(x, y)Z = himmelblau(X, Y)plt.figure(figsize=(12, 5))plt.subplot(1, 2, 1)plt.contour(X, Y, Z, 50, cmap='viridis')sgd_traj = np.array(sgd_trajectory)plt.plot(sgd_traj[:, 0], sgd_traj[:, 1], 'ro-', markersize=2, label='SGD')plt.title('SGD Optimization Path')plt.xlabel('x')plt.ylabel('y')plt.legend()plt.subplot(1, 2, 2)plt.contour(X, Y, Z, 50, cmap='viridis')momentum_traj = np.array(momentum_trajectory)plt.plot(momentum_traj[:, 0], momentum_traj[:, 1], 'bo-', markersize=2, label='Momentum SGD')plt.title('Momentum SGD Optimization Path')plt.xlabel('x')plt.ylabel('y')plt.legend()plt.tight_layout()plt.show()# 运行对比
compare_sgd_vs_momentum()

2.3 Nesterov加速梯度（NAG）

NAG是动量法的改进版本，它先根据动量方向进行预更新，然后计算梯度。

NAG更新公式：
v_t+1 = γv_t + η∇L(θ_t - γv_t)
θ_t+1 = θ_t - v_t+1

class NAGOptimizer:def __init__(self, params, lr=0.01, momentum=0.9):"""手动实现NAG优化器Args:params: 模型参数lr: 学习率momentum: 动量系数"""self.params = list(params)self.lr = lrself.momentum = momentumself.velocities = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""for i, param in enumerate(self.params):if param.grad is not None:# 保存当前参数current_param = param.data.clone()# 预更新参数param.data = param.data - self.momentum * self.velocities[i]# 这里应该重新计算梯度，但为了简化，我们使用已计算的梯度# 在实际实现中，需要在前向传播前进行预更新# 恢复参数param.data = current_param# 更新速度self.velocities[i] = self.momentum * self.velocities[i] + self.lr * param.grad.data# 更新参数param.data = param.data - self.velocities[i]# NAG与动量SGD对比
def compare_momentum_vs_nag():"""对比动量SGD和NAG的优化路径"""# 定义测试函数def rastrigin(x, y):"""Rastrigin函数，多局部极小值点"""A = 10return A * 2 + (x**2 - A * np.cos(2 * np.pi * x)) + (y**2 - A * np.cos(2 * np.pi * y))def rastrigin_gradient(x, y):dx = 2 * x + 10 * 2 * np.pi * np.sin(2 * np.pi * x)dy = 2 * y + 10 * 2 * np.pi * np.sin(2 * np.pi * y)return np.array([dx, dy])# 初始化参数x_momentum, y_momentum = 2.5, 2.5x_nag, y_nag = 2.5, 2.5lr = 0.01momentum = 0.9# 记录轨迹momentum_trajectory = [(x_momentum, y_momentum)]nag_trajectory = [(x_nag, y_nag)]# 速度初始化vx_m, vy_m = 0.0, 0.0vx_n, vy_n = 0.0, 0.0# 优化过程for i in range(500):# Momentum SGDgrad_m = rastrigin_gradient(x_momentum, y_momentum)vx_m = momentum * vx_m + lr * grad_m[0]vy_m = momentum * vy_m + lr * grad_m[1]x_momentum = x_momentum - vx_my_momentum = y_momentum - vy_mmomentum_trajectory.append((x_momentum, y_momentum))# NAG# 预更新位置x_nag_lookahead = x_nag - momentum * vx_ny_nag_lookahead = y_nag - momentum * vy_n# 计算预更新位置的梯度grad_n = rastrigin_gradient(x_nag_lookahead, y_nag_lookahead)# 更新速度vx_n = momentum * vx_n + lr * grad_n[0]vy_n = momentum * vy_n + lr * grad_n[1]# 更新参数x_nag = x_nag - vx_ny_nag = y_nag - vy_nnag_trajectory.append((x_nag, y_nag))# 可视化x = np.linspace(-3, 3, 100)y = np.linspace(-3, 3, 100)X, Y = np.meshgrid(x, y)Z = rastrigin(X, Y)plt.figure(figsize=(12, 5))plt.subplot(1, 2, 1)plt.contour(X, Y, Z, 50, cmap='viridis', levels=np.logspace(-1, 3, 20))momentum_traj = np.array(momentum_trajectory)plt.plot(momentum_traj[:, 0], momentum_traj[:, 1], 'ro-', markersize=2, label='Momentum SGD')plt.title('Momentum SGD Optimization Path')plt.xlabel('x')plt.ylabel('y')plt.legend()plt.subplot(1, 2, 2)plt.contour(X, Y, Z, 50, cmap='viridis', levels=np.logspace(-1, 3, 20))nag_traj = np.array(nag_trajectory)plt.plot(nag_traj[:, 0], nag_traj[:, 1], 'bo-', markersize=2, label='NAG')plt.title('NAG Optimization Path')plt.xlabel('x')plt.ylabel('y')plt.legend()plt.tight_layout()plt.show()# 运行对比
compare_momentum_vs_nag()

第三部分：自适应学习率优化器

3.1 AdaGrad

AdaGrad为每个参数自适应地调整学习率，适合处理稀疏数据。

AdaGrad更新公式：
g_t,i = ∇L(θ_t,i)
G_t,ii = G_t-1,ii + g_t,i²
θ_t+1,i = θ_t,i - (η/√(G_t,ii + ε)) · g_t,i

class AdaGradOptimizer:def __init__(self, params, lr=0.01, epsilon=1e-8):"""手动实现AdaGrad优化器Args:params: 模型参数lr: 学习率epsilon: 小常数，防止除零"""self.params = list(params)self.lr = lrself.epsilon = epsilonself.G = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""for i, param in enumerate(self.params):if param.grad is not None:# 累积平方梯度self.G[i] = self.G[i] + param.grad.data ** 2# 计算自适应学习率adaptive_lr = self.lr / (torch.sqrt(self.G[i]) + self.epsilon)# 更新参数param.data = param.data - adaptive_lr * param.grad.data# AdaGrad优化器测试
def test_adagrad_optimizer():"""测试AdaGrad优化器在不同稀疏性数据上的表现"""# 创建测试数据torch.manual_seed(42)# 密集数据dense_features = torch.randn(1000, 50)dense_targets = torch.randn(1000, 1)# 稀疏数据（90%为零）sparse_features = dense_features.clone()mask = torch.rand_like(sparse_features) > 0.9sparse_features = sparse_features * mask.float()# 创建简单模型model_dense = nn.Linear(50, 1)model_sparse = nn.Linear(50, 1)# 使用相同的初始权重model_sparse.load_state_dict(model_dense.state_dict())# 优化器optimizer_dense = AdaGradOptimizer(model_dense.parameters(), lr=0.1)optimizer_sparse = AdaGradOptimizer(model_sparse.parameters(), lr=0.1)# 训练记录losses_dense = []losses_sparse = []# 训练过程for epoch in range(100):# 密集数据训练outputs_dense = model_dense(dense_features)loss_dense = nn.MSELoss()(outputs_dense, dense_targets)optimizer_dense.zero_grad()loss_dense.backward()optimizer_dense.step()losses_dense.append(loss_dense.item())# 稀疏数据训练outputs_sparse = model_sparse(sparse_features)loss_sparse = nn.MSELoss()(outputs_sparse, dense_targets)optimizer_sparse.zero_grad()loss_sparse.backward()optimizer_sparse.step()losses_sparse.append(loss_sparse.item())# 绘制训练曲线plt.figure(figsize=(10, 6))plt.plot(losses_dense, label='Dense Data')plt.plot(losses_sparse, label='Sparse Data')plt.xlabel('Epoch')plt.ylabel('Loss')plt.title('AdaGrad Performance on Different Data Sparsity')plt.legend()plt.grid(True)plt.show()# 运行测试
test_adagrad_optimizer()

3.2 RMSProp

RMSProp改进了AdaGrad，通过指数移动平均来累积梯度，避免学习率过早衰减。

RMSProp更新公式：
E[g²]_t = βE[g²]_t-1 + (1-β)g_t²
θ_t+1 = θ_t - (η/√(E[g²]_t + ε)) · g_t

class RMSPropOptimizer:def __init__(self, params, lr=0.01, alpha=0.99, epsilon=1e-8):"""手动实现RMSProp优化器Args:params: 模型参数lr: 学习率alpha: 衰减率epsilon: 小常数，防止除零"""self.params = list(params)self.lr = lrself.alpha = alphaself.epsilon = epsilonself.avg_sq_grad = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""for i, param in enumerate(self.params):if param.grad is not None:# 计算指数移动平均的平方梯度self.avg_sq_grad[i] = self.alpha * self.avg_sq_grad[i] + (1 - self.alpha) * param.grad.data ** 2# 计算自适应学习率adaptive_lr = self.lr / (torch.sqrt(self.avg_sq_grad[i]) + self.epsilon)# 更新参数param.data = param.data - adaptive_lr * param.grad.data# RMSProp与AdaGrad对比
def compare_adagrad_vs_rmsprop():"""对比AdaGrad和RMSProp在非平稳问题上的表现"""# 定义非平稳目标函数def non_stationary_function(x, iteration):"""随时间变化的非平稳函数"""return (x - 0.5 * np.sin(iteration / 50)) ** 2def non_stationary_gradient(x, iteration):"""非平稳函数的梯度"""return 2 * (x - 0.5 * np.sin(iteration / 50))# 初始化参数x_adagrad, x_rmsprop = 2.0, 2.0lr = 0.1epsilon = 1e-8# AdaGrad变量G_adagrad = 0# RMSProp变量avg_sq_grad = 0alpha = 0.99# 记录轨迹adagrad_trajectory = [x_adagrad]rmsprop_trajectory = [x_rmsprop]target_trajectory = [0.5 * np.sin(0)]  # 初始目标值# 优化过程for iteration in range(1, 1000):# 当前目标值current_target = 0.5 * np.sin(iteration / 50)target_trajectory.append(current_target)# AdaGradgrad_adagrad = non_stationary_gradient(x_adagrad, iteration)G_adagrad = G_adagrad + grad_adagrad ** 2adaptive_lr_adagrad = lr / (np.sqrt(G_adagrad) + epsilon)x_adagrad = x_adagrad - adaptive_lr_adagrad * grad_adagradadagrad_trajectory.append(x_adagrad)# RMSPropgrad_rmsprop = non_stationary_gradient(x_rmsprop, iteration)avg_sq_grad = alpha * avg_sq_grad + (1 - alpha) * grad_rmsprop ** 2adaptive_lr_rmsprop = lr / (np.sqrt(avg_sq_grad) + epsilon)x_rmsprop = x_rmsprop - adaptive_lr_rmsprop * grad_rmsproprmsprop_trajectory.append(x_rmsprop)# 可视化plt.figure(figsize=(12, 6))iterations = range(1000)plt.plot(iterations, adagrad_trajectory, 'r-', label='AdaGrad', alpha=0.7)plt.plot(iterations, rmsprop_trajectory, 'b-', label='RMSProp', alpha=0.7)plt.plot(iterations, target_trajectory, 'g--', label='Moving Target', alpha=0.7)plt.xlabel('Iteration')plt.ylabel('Parameter Value')plt.title('AdaGrad vs RMSProp on Non-stationary Problem')plt.legend()plt.grid(True)plt.show()# 运行对比
compare_adagrad_vs_rmsprop()

3.3 Adam（Adaptive Moment Estimation）

Adam结合了动量法和RMSProp的优点，是当前最流行的优化器之一。

Adam更新公式：
m_t = β₁m_t-1 + (1-β₁)g_t # 一阶矩估计
v_t = β₂v_t-1 + (1-β₂)g_t² # 二阶矩估计
m̂_t = m_t/(1-β₁ᵗ) # 偏差校正
v̂_t = v_t/(1-β₂ᵗ) # 偏差校正
θ_t+1 = θ_t - (η/√(v̂_t + ε)) · m̂_t

class AdamOptimizer:def __init__(self, params, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):"""手动实现Adam优化器Args:params: 模型参数lr: 学习率beta1: 一阶矩衰减率beta2: 二阶矩衰减率epsilon: 小常数，防止除零"""self.params = list(params)self.lr = lrself.beta1 = beta1self.beta2 = beta2self.epsilon = epsilonself.t = 0# 初始化矩估计self.m = [torch.zeros_like(param) for param in self.params]self.v = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""self.t += 1for i, param in enumerate(self.params):if param.grad is not None:# 更新一阶矩估计self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * param.grad.data# 更新二阶矩估计self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * param.grad.data ** 2# 偏差校正m_hat = self.m[i] / (1 - self.beta1 ** self.t)v_hat = self.v[i] / (1 - self.beta2 ** self.t)# 更新参数param.data = param.data - self.lr * m_hat / (torch.sqrt(v_hat) + self.epsilon)# Adam优化器全面测试
def comprehensive_adam_test():"""全面测试Adam优化器在不同场景下的表现"""# 测试1: 不同学习率的影响print("测试不同学习率对Adam的影响...")# 定义测试函数def test_function(x):return x ** 4 - 3 * x ** 3 + 2 * x ** 2 - x + 5def test_gradient(x):return 4 * x ** 3 - 9 * x ** 2 + 4 * x - 1# 测试不同学习率learning_rates = [0.001, 0.01, 0.1, 0.5]results = {}for lr in learning_rates:x = torch.tensor([5.0], requires_grad=True)  # 初始点optimizer = AdamOptimizer([x], lr=lr)trajectory = [x.item()]for step in range(1000):# 计算损失loss = test_function(x)# 反向传播optimizer.zero_grad()loss.backward()optimizer.step()trajectory.append(x.item())results[f'LR={lr}'] = trajectory# 可视化不同学习率的效果plt.figure(figsize=(12, 6))for label, trajectory in results.items():plt.plot(trajectory, label=label)plt.xlabel('Iteration')plt.ylabel('Parameter Value')plt.title('Adam with Different Learning Rates')plt.legend()plt.grid(True)plt.show()return results# 运行全面测试
adam_results = comprehensive_adam_test()

3.4 AdamW：解耦权重衰减

AdamW改进了Adam，将权重衰减与梯度更新解耦，解决了Adam在某些任务上的泛化问题。

AdamW更新公式：
与Adam相同，但权重衰减单独处理：
θ_t+1 = θ_t - (η/√(v̂_t + ε)) · m̂_t - ηλθ_t

class AdamWOptimizer:def __init__(self, params, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, weight_decay=0.01):"""手动实现AdamW优化器Args:params: 模型参数lr: 学习率beta1: 一阶矩衰减率beta2: 二阶矩衰减率epsilon: 小常数，防止除零weight_decay: 权重衰减系数"""self.params = list(params)self.lr = lrself.beta1 = beta1self.beta2 = beta2self.epsilon = epsilonself.weight_decay = weight_decayself.t = 0# 初始化矩估计self.m = [torch.zeros_like(param) for param in self.params]self.v = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""self.t += 1for i, param in enumerate(self.params):if param.grad is not None:# 应用权重衰减（解耦）param.data = param.data - self.lr * self.weight_decay * param.data# 更新一阶矩估计self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * param.grad.data# 更新二阶矩估计self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * param.grad.data ** 2# 偏差校正m_hat = self.m[i] / (1 - self.beta1 ** self.t)v_hat = self.v[i] / (1 - self.beta2 ** self.t)# 更新参数param.data = param.data - self.lr * m_hat / (torch.sqrt(v_hat) + self.epsilon)# Adam vs AdamW对比
def compare_adam_vs_adamw():"""对比Adam和AdamW在深度学习任务上的表现"""# 使用CIFAR-10数据集transform = transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True)# 创建相同结构的两个模型model_adam = torchvision.models.resnet18(pretrained=False, num_classes=10)model_adamw = torchvision.models.resnet18(pretrained=False, num_classes=10)# 使用相同的初始权重model_adamw.load_state_dict(model_adam.state_dict())# 优化器optimizer_adam = AdamOptimizer(model_adam.parameters(), lr=0.001)optimizer_adamw = AdamWOptimizer(model_adamw.parameters(), lr=0.001, weight_decay=0.01)criterion = nn.CrossEntropyLoss()# 训练记录losses_adam = []losses_adamw = []accuracies_adam = []accuracies_adamw = []# 训练过程for epoch in range(10):  # 简化训练轮数# Adam训练model_adam.train()running_loss_adam = 0.0correct_adam = 0total_adam = 0for i, (inputs, labels) in enumerate(trainloader):optimizer_adam.zero_grad()outputs = model_adam(inputs)loss = criterion(outputs, labels)loss.backward()optimizer_adam.step()running_loss_adam += loss.item()_, predicted = outputs.max(1)total_adam += labels.size(0)correct_adam += predicted.eq(labels).sum().item()losses_adam.append(running_loss_adam / len(trainloader))accuracies_adam.append(100. * correct_adam / total_adam)# AdamW训练model_adamw.train()running_loss_adamw = 0.0correct_adamw = 0total_adamw = 0for i, (inputs, labels) in enumerate(trainloader):optimizer_adamw.zero_grad()outputs = model_adamw(inputs)loss = criterion(outputs, labels)loss.backward()optimizer_adamw.step()running_loss_adamw += loss.item()_, predicted = outputs.max(1)total_adamw += labels.size(0)correct_adamw += predicted.eq(labels).sum().item()losses_adamw.append(running_loss_adamw / len(trainloader))accuracies_adamw.append(100. * correct_adamw / total_adamw)print(f'Epoch {epoch+1}: Adam Loss: {losses_adam[-1]:.4f}, Acc: {accuracies_adam[-1]:.2f}% | 'f'AdamW Loss: {losses_adamw[-1]:.4f}, Acc: {accuracies_adamw[-1]:.2f}%')# 可视化结果plt.figure(figsize=(12, 5))plt.subplot(1, 2, 1)plt.plot(losses_adam, 'b-', label='Adam')plt.plot(losses_adamw, 'r-', label='AdamW')plt.xlabel('Epoch')plt.ylabel('Loss')plt.title('Training Loss Comparison')plt.legend()plt.grid(True)plt.subplot(1, 2, 2)plt.plot(accuracies_adam, 'b-', label='Adam')plt.plot(accuracies_adamw, 'r-', label='AdamW')plt.xlabel('Epoch')plt.ylabel('Accuracy (%)')plt.title('Training Accuracy Comparison')plt.legend()plt.grid(True)plt.tight_layout()plt.show()return losses_adam, losses_adamw, accuracies_adam, accuracies_adamw# 运行对比
adam_loss, adamw_loss, adam_acc, adamw_acc = compare_adam_vs_adamw()

第四部分：优化器选择指南与实战建议

4.1 优化器性能综合对比

def comprehensive_optimizer_comparison():"""综合对比各种优化器的性能"""# 定义复杂的测试函数def complex_function(x, y):"""具有多个局部极小值的复杂函数"""return (1.5 - x + x * y) ** 2 + (2.25 - x + x * y ** 2) ** 2 + (2.625 - x + x * y ** 3) ** 2def complex_gradient(x, y):"""复杂函数的梯度"""term1 = 2 * (1.5 - x + x * y) * (-1 + y)term2 = 2 * (2.25 - x + x * y ** 2) * (-1 + y ** 2)term3 = 2 * (2.625 - x + x * y ** 3) * (-1 + y ** 3)dx = term1 + term2 + term3term1 = 2 * (1.5 - x + x * y) * xterm2 = 2 * (2.25 - x + x * y ** 2) * (2 * x * y)term3 = 2 * (2.625 - x + x * y ** 3) * (3 * x * y ** 2)dy = term1 + term2 + term3return np.array([dx, dy])# 优化器列表optimizers = {'SGD': {'lr': 0.01},'Momentum': {'lr': 0.01, 'momentum': 0.9},'AdaGrad': {'lr': 0.1},'RMSProp': {'lr': 0.01, 'alpha': 0.99},'Adam': {'lr': 0.001, 'beta1': 0.9, 'beta2': 0.999}}# 初始化参数initial_point = np.array([-1.0, -1.5])trajectories = {}final_values = {}# 对每个优化器进行测试for opt_name, opt_params in optimizers.items():print(f"测试 {opt_name} 优化器...")# 初始化参数params = initial_point.copy()trajectory = [params.copy()]# 优化器特定的变量初始化if opt_name == 'SGD':# 无需额外变量passelif opt_name == 'Momentum':velocity = np.zeros_like(params)elif opt_name == 'AdaGrad':G = np.zeros_like(params)elif opt_name == 'RMSProp':avg_sq_grad = np.zeros_like(params)elif opt_name == 'Adam':m = np.zeros_like(params)v = np.zeros_like(params)t = 0# 优化过程for iteration in range(1000):grad = complex_gradient(params[0], params[1])if opt_name == 'SGD':update = opt_params['lr'] * gradelif opt_name == 'Momentum':velocity = opt_params['momentum'] * velocity + opt_params['lr'] * gradupdate = velocityelif opt_name == 'AdaGrad':G = G + grad ** 2update = opt_params['lr'] * grad / (np.sqrt(G) + 1e-8)elif opt_name == 'RMSProp':avg_sq_grad = opt_params['alpha'] * avg_sq_grad + (1 - opt_params['alpha']) * grad ** 2update = opt_params['lr'] * grad / (np.sqrt(avg_sq_grad) + 1e-8)elif opt_name == 'Adam':t += 1m = opt_params['beta1'] * m + (1 - opt_params['beta1']) * gradv = opt_params['beta2'] * v + (1 - opt_params['beta2']) * grad ** 2# 偏差校正m_hat = m / (1 - opt_params['beta1'] ** t)v_hat = v / (1 - opt_params['beta2'] ** t)update = opt_params['lr'] * m_hat / (np.sqrt(v_hat) + 1e-8)# 更新参数params = params - updatetrajectory.append(params.copy())trajectories[opt_name] = np.array(trajectory)final_values[opt_name] = complex_function(params[0], params[1])print(f"  {opt_name} 最终损失: {final_values[opt_name]:.6f}")# 可视化结果x = np.linspace(-4.5, 4.5, 100)y = np.linspace(-4.5, 4.5, 100)X, Y = np.meshgrid(x, y)Z = complex_function(X, Y)plt.figure(figsize=(15, 12))# 绘制等高线图contour = plt.contour(X, Y, Z, 50, cmap='viridis', alpha=0.6)plt.colorbar(contour)# 绘制各优化器的轨迹colors = ['red', 'blue', 'green', 'orange', 'purple']for i, (opt_name, trajectory) in enumerate(trajectories.items()):plt.plot(trajectory[:, 0], trajectory[:, 1], color=colors[i], marker='o', markersize=2, linewidth=1.5, label=opt_name)plt.title('Optimizer Trajectories on Complex Function')plt.xlabel('x')plt.ylabel('y')plt.legend()plt.grid(True)plt.show()# 绘制收敛曲线plt.figure(figsize=(12, 6))for opt_name, trajectory in trajectories.items():losses = [complex_function(p[0], p[1]) for p in trajectory]plt.plot(losses, label=opt_name, linewidth=2)plt.yscale('log')plt.xlabel('Iteration')plt.ylabel('Loss (log scale)')plt.title('Optimizer Convergence Comparison')plt.legend()plt.grid(True)plt.show()return trajectories, final_values# 运行综合对比
trajectories, final_values = comprehensive_optimizer_comparison()

4.2 优化器选择指南

根据理论分析和实验结果，我们总结出以下优化器选择指南：

优化器	适用场景	优点	缺点	推荐参数
SGD	凸优化、简单问题	理论保证、可解释性强	收敛慢、易陷入局部最优	lr=0.01
Momentum	中等复杂度问题	加速收敛、减少振荡	需要调整动量参数	lr=0.01, momentum=0.9
AdaGrad	稀疏数据、自然语言处理	自适应学习率、适合稀疏特征	学习率过早衰减	lr=0.1
RMSProp	非平稳目标、RNN	处理非平稳问题效果好	对超参数敏感	lr=0.001, alpha=0.99
Adam	大多数深度学习任务	快速收敛、自适应学习率	可能泛化不如SGD	lr=0.001, beta1=0.9, beta2=0.999
AdamW	计算机视觉、需要正则化的任务	更好的泛化性能	超参数更多	lr=0.001, weight_decay=0.01

4.3 实战建议与调参技巧

def practical_optimizer_tips():"""优化器实战建议和调参技巧"""tips = {'学习率设置': ['从默认学习率开始（Adam: 0.001, SGD: 0.01）','使用学习率预热（warmup）策略','实施学习率衰减（step decay或cosine annealing）','对于小批量数据，使用较小的学习率'],'批量大小': ['大批量：使用更大学习率，加快训练速度','小批量：使用更小学习率，可能获得更好泛化','常见批量大小：32, 64, 128, 256'],'优化器选择': ['CV任务：Adam/AdamW通常是不错的选择','NLP任务：AdaGrad/Adam适合稀疏特征','简单任务：SGD可能获得更好泛化','不稳定训练：尝试减小学习率或换用SGD'],'超参数调优': ['使用网格搜索或随机搜索','关注验证集性能而非训练集损失','考虑使用自动化超参数优化工具（如Optuna）'],'高级技巧': ['梯度裁剪（clip gradients）防止梯度爆炸','使用SWA（Stochastic Weight Averaging）提高泛化','尝试Lion、AdaBelief等新兴优化器']}print("优化器实战建议与调参技巧")print("=" * 50)for category, advice_list in tips.items():print(f"\n{category}:")for i, advice in enumerate(advice_list, 1):print(f"  {i}. {advice}")return tips# 学习率搜索示例
def learning_rate_search_example():"""学习率搜索的代码示例"""# 定义测试函数def test_model():return nn.Sequential(nn.Linear(10, 50),nn.ReLU(),nn.Linear(50, 1))# 测试不同学习率learning_rates = [0.1, 0.01, 0.001, 0.0001]results = {}for lr in learning_rates:model = test_model()optimizer = AdamOptimizer(model.parameters(), lr=lr)# 简单训练循环losses = []for step in range(100):# 模拟数据inputs = torch.randn(32, 10)targets = torch.randn(32, 1)# 前向传播outputs = model(inputs)loss = nn.MSELoss()(outputs, targets)# 反向传播optimizer.zero_grad()loss.backward()optimizer.step()losses.append(loss.item())results[lr] = losses# 可视化结果plt.figure(figsize=(10, 6))for lr, losses in results.items():plt.plot(losses, label=f'LR={lr}')plt.xlabel('Step')plt.ylabel('Loss')plt.title('Learning Rate Search Results')plt.legend()plt.yscale('log')plt.grid(True)plt.show()return results# 运行示例
lr_results = learning_rate_search_example()
practical_tips = practical_optimizer_tips()

第五部分：新兴优化器与未来展望

5.1 新兴优化器介绍

class LionOptimizer:def __init__(self, params, lr=0.0001, beta1=0.9, beta2=0.99, weight_decay=0.0):"""Lion优化器实现（EvoLved Sign Momentum）参考：https://arxiv.org/abs/2302.06675Args:params: 模型参数lr: 学习率beta1: 一阶矩衰减率beta2: 二阶矩衰减率weight_decay: 权重衰减系数"""self.params = list(params)self.lr = lrself.beta1 = beta1self.beta2 = beta2self.weight_decay = weight_decayself.t = 0# 初始化矩估计self.m = [torch.zeros_like(param) for param in self.params]def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""self.t += 1for i, param in enumerate(self.params):if param.grad is not None:# 更新矩估计self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * param.grad.data# 计算更新方向（使用符号函数）update = torch.sign(self.m[i])# 应用权重衰减if self.weight_decay > 0:param.data = param.data - self.lr * self.weight_decay * param.data# 更新参数param.data = param.data - self.lr * updateclass AdaBeliefOptimizer:def __init__(self, params, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):"""AdaBelief优化器实现参考：https://arxiv.org/abs/2010.07468Args:params: 模型参数lr: 学习率beta1: 一阶矩衰减率beta2: 二阶矩衰减率epsilon: 小常数，防止除零"""self.params = list(params)self.lr = lrself.beta1 = beta1self.beta2 = beta2self.epsilon = epsilonself.t = 0# 初始化矩估计self.m = [torch.zeros_like(param) for param in self.params]self.s = [torch.zeros_like(param) for param in self.params]  # 二阶矩估计def zero_grad(self):"""清零梯度"""for param in self.params:if param.grad is not None:param.grad.zero_()def step(self):"""执行参数更新"""self.t += 1for i, param in enumerate(self.params):if param.grad is not None:# 更新一阶矩估计self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * param.grad.data# 更新二阶矩估计（基于梯度与一阶矩的偏差）grad_diff = param.grad.data - self.m[i]self.s[i] = self.beta2 * self.s[i] + (1 - self.beta2) * (grad_diff ** 2)# 偏差校正m_hat = self.m[i] / (1 - self.beta1 ** self.t)s_hat = self.s[i] / (1 - self.beta2 ** self.t)# 更新参数param.data = param.data - self.lr * m_hat / (torch.sqrt(s_hat) + self.epsilon)# 测试新兴优化器
def test_emerging_optimizers():"""测试新兴优化器的性能"""# 定义测试函数def difficult_function(x, y):"""难以优化的函数"""return torch.sin(5 * x) * torch.cos(5 * y) + 0.5 * (x ** 2 + y ** 2)# 初始化参数x_lion = torch.tensor([2.0], requires_grad=True)x_adabelief = torch.tensor([2.0], requires_grad=True)# 优化器lion_optimizer = LionOptimizer([x_lion], lr=0.0001)adabelief_optimizer = AdaBeliefOptimizer([x_adabelief], lr=0.001)# 训练记录lion_losses = []adabelief_losses = []# 优化过程for step in range(1000):# Lion优化器lion_loss = difficult_function(x_lion, torch.tensor([1.0]))lion_optimizer.zero_grad()lion_loss.backward()lion_optimizer.step()lion_losses.append(lion_loss.item())# AdaBelief优化器adabelief_loss = difficult_function(x_adabelief, torch.tensor([1.0]))adabelief_optimizer.zero_grad()adabelief_loss.backward()adabelief_optimizer.step()adabelief_losses.append(adabelief_loss.item())# 可视化结果plt.figure(figsize=(10, 6))plt.plot(lion_losses, label='Lion')plt.plot(adabelief_losses, label='AdaBelief')plt.xlabel('Step')plt.ylabel('Loss')plt.title('Emerging Optimizers Performance')plt.legend()plt.grid(True)plt.yscale('log')plt.show()return lion_losses, adabelief_losses# 运行测试
lion_loss, adabelief_loss = test_emerging_optimizers()