class Model(torch.nn.Module): def __init__(self, X, y, beta=None): super().__init__() self.X = X self.y = y if beta is None: beta = torch.zeros(X.shape[1]) beta.requires_grad = True self.beta = torch.nn.Parameter(beta) def forward(self, X): return X @ self.beta def fit(self, opt, n=1000, loss_fn = torch.nn.MSELoss()): losses = [] for i in range(n): loss = loss_fn( self(self.X).squeeze(), self.y.squeeze() ) loss.backward() opt.step() opt.zero_grad() losses.append(loss.item()) return lossesclass Model(torch.nn.Module): def __init__(self, X, y, beta=None): super().__init__() self.X = X self.y = y if beta is None: beta = torch.zeros(X.shape[1]) beta.requires_grad = True self.beta = torch.nn.Parameter(beta) def forward(self, X): return X @ self.beta def fit(self, opt, n=1000, loss_fn = torch.nn.MSELoss()): losses = [] for i in range(n): loss = loss_fn( self(self.X).squeeze(), self.y.squeeze() ) loss.backward() opt.step() opt.zero_grad() losses.append(loss.item()) return losses
pytorch - optim & nn
Lecture 23
Dr. Colin Rundel
Odds & Ends
Torch models
Implementation details:
Models are implemented as a class - inherits from
torch.nn.ModuleMust implement constructor and
forward()method__init__()should call parent constructor viasuper()- Use
torch.nn.Parameter()to indicate model parameters
- Use
forward()should implement the model - constants + parameters -> predictions
Fitting proceedure:
For each iteration of solver:
Get current predictions via a call to
forward()or equivalent.Calculate a loss or equivalent (scalar)
Call
backward()method on lossUse built-in optimizer (
step()andzero_grad())
From last time
What is self(self.X)?
This is (mostly) just short hand for calling self.forward(X) to generate the output tensors from the current value(s) of the parameters. This is done via the special __call__() method in the torch.nn.Module class. __call__() allows python classes to be invoked like functions.
MNIST & Logistic models
MNIST handwritten digits - simplified
(1797, 64)array([[ 0., 0., 5., 13., 9., 1., 0., 0., 0., 0.,
13., 15., 10., 15., 5., 0., 0., 3., 15., 2.,
0., 11., 8., 0., 0., 4., 12., 0., 0., 8.,
8., 0., 0., 5., 8., 0., 0., 9., 8., 0.,
0., 4., 11., 0., 1., 12., 7., 0., 0., 2.,
14., 5., 10., 12., 0., 0., 0., 0., 6., 13.,
10., 0., 0., 0.],
[ 0., 0., 0., 12., 13., 5., 0., 0., 0., 0.,
0., 11., 16., 9., 0., 0., 0., 0., 3., 15.,
16., 6., 0., 0., 0., 7., 15., 16., 16., 2.,
0., 0., 0., 0., 1., 16., 16., 3., 0., 0.,
0., 0., 1., 16., 16., 6., 0., 0., 0., 0.,
1., 16., 16., 6., 0., 0., 0., 0., 0., 11.,
16., 10., 0., 0.]])Example digits
Test train split
As Tensors
PyTorch Model
class mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses = [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) return lossesclass mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses = [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) return lossesclass mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses = [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) return lossesclass mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses = [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) return losses
Cross entropy loss
Out-of-sample accuracy
tensor([[-12.0853, -12.4920, -16.3669, ..., 96.2283, 32.7762, 63.1954],
[-41.2113, 26.0970, -6.4925, ..., -12.4305, -13.3414, 36.3478],
[-32.5302, 11.3226, 13.1565, ..., 112.7131, 40.7483, 55.6575],
...,
[ 19.0148, 15.7732, -21.7103, ..., -3.0289, 21.8787, -1.7871],
[-37.8395, 28.8395, -19.3067, ..., 53.3926, 28.4977, 58.1099],
[-34.6293, -8.0961, 19.3990, ..., -7.2846, 31.9065, 11.2288]],
grad_fn=<SqueezeBackward0>)tensor([7, 9, 7, 6, 0, 2, 4, 3, 6, 3, 7, 8, 7, 9, 4, 3, 1, 3, 8, 4, 0, 3, 9, 1,
3, 6, 6, 0, 4, 4, 1, 9, 1, 2, 3, 2, 7, 6, 4, 8, 6, 4, 4, 0, 9, 2, 8, 2,
4, 4, 4, 1, 7, 6, 8, 2, 9, 5, 5, 0, 1, 3, 1, 8, 8, 1, 3, 9, 1, 0, 9, 6,
9, 5, 8, 1, 9, 2, 1, 3, 8, 7, 3, 3, 1, 7, 7, 5, 8, 2, 1, 8, 9, 1, 6, 4,
5, 2, 2, 4, 5, 4, 7, 6, 5, 7, 2, 4, 1, 0, 7, 6, 1, 2, 9, 8, 2, 5, 0, 3,
2, 7, 6, 4, 6, 2, 1, 1, 6, 9, 6, 8, 3, 4, 7, 5, 0, 9, 1, 0, 5, 6, 7, 6,
3, 8, 3, 2, 0, 4, 0, 9, 5, 4, 6, 4, 1, 9, 6, 2, 7, 9, 0, 7, 9, 3, 4, 1,
3, 8, 6, 4, 7, 1, 5, 7, 4, 7, 4, 4, 2, 2, 1, 1, 4, 4, 3, 5, 5, 9, 4, 5,
5, 9, 3, 9, 3, 1, 2, 0, 8, 2, 8, 5, 2, 4, 6, 8, 3, 9, 1, 0, 8, 1, 8, 5,
6, 8, 7, 1, 8, 0, 4, 9, 7, 0, 5, 5, 6, 1, 3, 4, 5, 8, 2, 0, 9, 6, 6, 7,
8, 4, 1, 0, 5, 1, 5, 1, 6, 4, 7, 1, 2, 6, 4, 4, 6, 3, 2, 5, 2, 6, 5, 2,
4, 4, 7, 0, 1, 0, 4, 3, 1, 2, 7, 9, 8, 5, 9, 5, 7, 0, 0, 1, 4, 9, 4, 0,
7, 7, 7, 5, 3, 5, 3, 8, 7, 5, 8, 3, 7, 0, 8, 9, 1, 7, 9, 8, 5, 0, 2, 0,
9, 7, 0, 9, 5, 5, 9, 6, 1, 2, 3, 9, 1, 3, 2, 9, 3, 0, 3, 4, 1, 8, 1, 8,
5, 0, 9, 2, 7, 2, 3, 5, 2, 6, 3, 4, 1, 5, 0, 5, 4, 6, 3, 2, 5, 8, 9, 3])Calculating Accuracy
class mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.beta = torch.nn.Parameter( torch.randn(input_dim, output_dim, requires_grad=True) ) self.intercept = torch.nn.Parameter( torch.randn(output_dim, requires_grad=True) ) def forward(self, X): return (X @ self.beta + self.intercept).squeeze() def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
Performance
NN Layers
class mnist_nn_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.linear = torch.nn.Linear(input_dim, output_dim) def forward(self, X): return self.linear(X) def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_nn_model(torch.nn.Module): def __init__(self, input_dim, output_dim): super().__init__() self.linear = torch.nn.Linear(input_dim, output_dim) def forward(self, X): return self.linear(X) def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
NN linear layer
Applies a linear transform to the incoming data (\(x\)): \[y = x A^T+b\]
Performance
Feedforward Neural Network
FNN Model
class mnist_fnn_model(torch.nn.Module): def __init__(self, input_dim, hidden_dim, output_dim, nl_step = torch.nn.ReLU(), seed=1234): super().__init__() self.l1 = torch.nn.Linear(input_dim, hidden_dim) self.nl = nl_step self.l2 = torch.nn.Linear(hidden_dim, output_dim) def forward(self, X): out = self.l1(X) out = self.nl(out) out = self.l2(out) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_fnn_model(torch.nn.Module): def __init__(self, input_dim, hidden_dim, output_dim, nl_step = torch.nn.ReLU(), seed=1234): super().__init__() self.l1 = torch.nn.Linear(input_dim, hidden_dim) self.nl = nl_step self.l2 = torch.nn.Linear(hidden_dim, output_dim) def forward(self, X): out = self.l1(X) out = self.nl(out) out = self.l2(out) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
Non-linear activation functions
\[\text{Tanh}(x) = \frac{\exp(x)-\exp(-x)}{\exp(x) + \exp(-x)}\]
\[\text{ReLU}(x) = \max(0,x)\]
Model parameters
Performance - ReLU
Performance - tanh
loss, train_acc, test_acc = mnist_fnn_model(
64,64,10, nl_step=torch.nn.Tanh()
).fit(
X_train, y_train, X_test, y_test, n=2000
)
train_acc[-5:][tensor(0.9944), tensor(0.9944), tensor(0.9944), tensor(0.9951), tensor(0.9951)][tensor(0.9722), tensor(0.9722), tensor(0.9722), tensor(0.9722), tensor(0.9722)]
Adding another layer
class mnist_fnn2_model(torch.nn.Module): def __init__(self, input_dim, hidden_dim, output_dim, nl_step = torch.nn.ReLU(), seed=1234): super().__init__() self.l1 = torch.nn.Linear(input_dim, hidden_dim) self.nl = nl_step self.l2 = torch.nn.Linear(hidden_dim, hidden_dim) self.nl = nl_step self.l3 = torch.nn.Linear(hidden_dim, output_dim) def forward(self, X): out = self.l1(X) out = self.nl(out) out = self.l2(out) out = self.nl(out) out = self.l3(out) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): loss_fn = torch.nn.CrossEntropyLoss() opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = loss_fn(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_fnn2_model(torch.nn.Module): def __init__(self, input_dim, hidden_dim, output_dim, nl_step = torch.nn.ReLU(), seed=1234): super().__init__() self.l1 = torch.nn.Linear(input_dim, hidden_dim) self.nl = nl_step self.l2 = torch.nn.Linear(hidden_dim, hidden_dim) self.nl = nl_step self.l3 = torch.nn.Linear(hidden_dim, output_dim) def forward(self, X): out = self.l1(X) out = self.nl(out) out = self.l2(out) out = self.nl(out) out = self.l3(out) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): loss_fn = torch.nn.CrossEntropyLoss() opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = loss_fn(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
Performance - relu
loss, train_acc, test_acc = mnist_fnn2_model(
64,64,10, nl_step=torch.nn.ReLU()
).fit(
X_train, y_train, X_test, y_test, n=1000
)
train_acc[-5:][tensor(0.9889), tensor(0.9889), tensor(0.9889), tensor(0.9896), tensor(0.9896)][tensor(0.9639), tensor(0.9639), tensor(0.9639), tensor(0.9639), tensor(0.9639)]
Performance - tanh
loss, train_acc, test_acc = mnist_fnn2_model(
64,64,10, nl_step=torch.nn.Tanh()
).fit(
X_train, y_train, X_test, y_test, n=1000
)
train_acc[-5:][tensor(0.9833), tensor(0.9833), tensor(0.9840), tensor(0.9840), tensor(0.9840)][tensor(0.9667), tensor(0.9667), tensor(0.9667), tensor(0.9667), tensor(0.9667)]
Convolutional NN
2d convolutions
nn.Conv2d()
Parameter containing:
tensor([[[[-0.2126, -0.0904, 0.1491],
[-0.3284, -0.0247, 0.2515],
[ 0.1292, 0.2992, 0.2095]]],
[[[ 0.0066, 0.0926, 0.1636],
[ 0.1279, 0.0875, 0.1772],
[-0.2525, -0.3064, -0.1875]]],
[[[ 0.1264, -0.2146, 0.3322],
[ 0.2963, -0.2139, 0.0800],
[-0.0684, 0.3225, -0.2510]]],
[[[-0.3048, -0.1349, -0.2028],
[-0.2928, 0.2831, 0.0100],
[-0.3206, -0.1806, 0.1542]]]], requires_grad=True)Applying Conv2d()
tensor([[ 0., 0., 0., 10., 11., 0., 0., 0., 0., 0., 9., 16., 6., 0.,
0., 0., 0., 0., 15., 13., 0., 0., 0., 0., 0., 0., 14., 10.,
0., 0., 0., 0., 0., 1., 15., 12., 8., 2., 0., 0., 0., 0.,
12., 16., 16., 16., 10., 1., 0., 0., 7., 16., 12., 12., 16., 4.,
0., 0., 0., 9., 15., 12., 5., 0.]])torch.Size([1, 64])tensor([[[ 0.2514, 2.1372, 8.8115, 9.9783, 0.5591, -2.5852, 0.2514,
0.2514],
[ 0.2514, 5.6581, 12.7558, 4.9740, -6.5905, -4.0571, 0.2514,
0.2514],
[ 0.2514, 8.2995, 11.0063, -2.6590, -6.6684, -1.0240, 0.2514,
0.2514],
[ 0.4609, 9.4512, 10.1341, -1.7516, -1.4324, 1.8837, 0.5099,
0.2514],
[ 0.5029, 8.6012, 9.7387, 2.8518, 4.6978, 6.5248, 4.8637,
1.8431],
[ 0.4005, 6.8824, 9.6466, 5.0654, 3.8233, 3.7298, 1.7526,
0.2076],
[ 0.2514, 3.8012, 7.2894, 4.7997, 3.4224, 3.2588, -4.1872,
-6.6711],
[ 0.2514, 1.2950, 4.2678, 2.6568, -2.7527, -4.9629, -7.2134,
-5.1532]],
[[ 0.0478, -1.6394, -3.9379, -5.4284, -3.5895, -0.0607, 0.0478,
0.0478],
[ 0.0478, -1.1697, -1.7273, -1.3834, 0.4213, 0.8880, 0.0478,
0.0478],
[ 0.0478, 1.5533, 0.9504, -0.9727, -0.1531, 0.0877, 0.0478,
0.0478],
[ -0.1396, 1.8638, -0.5377, -4.9479, -4.4437, -2.5851, -0.4572,
0.0478],
[ 0.2250, 2.8338, -0.1295, -5.4801, -9.2396, -9.5720, -6.9883,
-2.7837],
[ 0.2114, 3.4083, 2.1474, -0.5831, -2.4905, -4.2029, -5.5236,
-3.8519],
[ 0.0478, 3.2514, 5.5371, 3.0785, 0.3570, 0.2892, 0.3249,
1.3401],
[ 0.0478, 1.1931, 4.9084, 6.9847, 7.8184, 7.7106, 4.2357,
1.1638]],
[[ 0.1583, -2.1009, -0.1559, 1.9367, 1.6083, 3.0070, 0.1583,
0.1583],
[ 0.1583, -2.8873, 4.4082, 4.5561, 1.6296, 3.3264, 0.1583,
0.1583],
[ 0.1583, 0.8331, 3.3770, 3.7860, 4.0609, 0.9167, 0.1583,
0.1583],
[ -0.0928, 2.8177, 0.8190, 2.1090, 6.0215, 0.2560, 0.0215,
0.1583],
[ 0.2382, 2.7821, -1.6238, 2.6216, 3.4753, 3.6559, 2.6303,
-0.2033],
[ 0.4904, 4.1289, -1.9946, 5.2175, 2.9831, 1.8904, 6.4272,
3.1027],
[ 0.1583, 4.7042, 0.4208, 2.3043, 8.4047, 5.9256, 1.6110,
4.7506],
[ 0.1583, 2.4835, 4.6907, 0.8699, 4.0090, 6.6918, 2.0562,
2.8038]],
[[ 0.2862, 1.6737, 1.2268, -1.6242, -5.7413, -4.8577, 0.2862,
0.2862],
[ 0.2862, 2.6882, 0.2595, -8.4971, -11.3987, -4.8227, 0.2862,
0.2862],
[ 0.2862, 0.7684, -0.7847, -12.8380, -12.4112, -1.5423, 0.2862,
0.2862],
[ 0.4404, -0.4851, -1.4913, -13.0500, -11.5873, -2.6399, -0.3550,
0.2862],
[ 0.2962, -0.2711, 0.7414, -10.5149, -9.5433, -7.9679, -7.0812,
-3.1006],
[ 0.0834, -1.6925, 0.2826, -9.6355, -10.2988, -6.0255, -8.2875,
-8.2106],
[ 0.2862, -2.0780, -1.0494, -5.4882, -14.9061, -14.9392, -9.8363,
-8.0511],
[ 0.2862, -1.1336, -3.8136, -3.7422, -6.9118, -9.1795, -8.4379,
-6.5932]]], grad_fn=<SqueezeBackward1>)Pooling
Convolutional model
class mnist_conv_model(torch.nn.Module): def __init__(self): super().__init__() self.cnn = torch.nn.Conv2d( in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=1 ) self.relu = torch.nn.ReLU() self.pool = torch.nn.MaxPool2d(kernel_size=2) self.lin = torch.nn.Linear(8 * 4 * 4, 10) def forward(self, X): out = self.cnn(X.view(-1, 1, 8, 8)) out = self.relu(out) out = self.pool(out) out = self.lin(out.view(-1, 8 * 4 * 4)) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): loss_fn = torch.nn.CrossEntropyLoss() opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = loss_fn(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_conv_model(torch.nn.Module): def __init__(self): super().__init__() self.cnn = torch.nn.Conv2d( in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=1 ) self.relu = torch.nn.ReLU() self.pool = torch.nn.MaxPool2d(kernel_size=2) self.lin = torch.nn.Linear(8 * 4 * 4, 10) def forward(self, X): out = self.cnn(X.view(-1, 1, 8, 8)) out = self.relu(out) out = self.pool(out) out = self.lin(out.view(-1, 8 * 4 * 4)) return out def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): loss_fn = torch.nn.CrossEntropyLoss() opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = loss_fn(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
Performance
Cleaning up models
class mnist_conv_model2(torch.nn.Module): def __init__(self): super().__init__() self.model = torch.nn.Sequential( torch.nn.Unflatten(1, (1,8,8)), torch.nn.Conv2d( in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=1 ), torch.nn.ReLU(), torch.nn.MaxPool2d(kernel_size=2), torch.nn.Flatten(), torch.nn.Linear(8 * 4 * 4, 10) ) def forward(self, X): return self.model(X) def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_conv_model2(torch.nn.Module): def __init__(self): super().__init__() self.model = torch.nn.Sequential( torch.nn.Unflatten(1, (1,8,8)), torch.nn.Conv2d( in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=1 ), torch.nn.ReLU(), torch.nn.MaxPool2d(kernel_size=2), torch.nn.Flatten(), torch.nn.Linear(8 * 4 * 4, 10) ) def forward(self, X): return self.model(X) def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)class mnist_conv_model2(torch.nn.Module): def __init__(self): super().__init__() self.model = torch.nn.Sequential( torch.nn.Unflatten(1, (1,8,8)), torch.nn.Conv2d( in_channels=1, out_channels=8, kernel_size=3, stride=1, padding=1 ), torch.nn.ReLU(), torch.nn.MaxPool2d(kernel_size=2), torch.nn.Flatten(), torch.nn.Linear(8 * 4 * 4, 10) ) def forward(self, X): return self.model(X) def fit(self, X_train, y_train, X_test, y_test, lr=0.001, n=1000, acc_step=10): opt = torch.optim.SGD(self.parameters(), lr=lr, momentum=0.9) losses, train_acc, test_acc = [], [], [] for i in range(n): opt.zero_grad() loss = torch.nn.CrossEntropyLoss()(self(X_train), y_train) loss.backward() opt.step() losses.append(loss.item()) if (i+1) % acc_step == 0: val, train_pred = torch.max(self(X_train), dim=1) val, test_pred = torch.max(self(X_test), dim=1) train_acc.append( (train_pred == y_train).sum() / len(y_train) ) test_acc.append( (test_pred == y_test).sum() / len(y_test) ) return (losses, train_acc, test_acc)
A bit more on non-linear
activation layers
Non-linear functions
Linear regression
class lin_reg(torch.nn.Module):
def __init__(self, X):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, self.p)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesModel results
Training loss:
Predictions
Double linear regression
class dbl_lin_reg(torch.nn.Module):
def __init__(self, X, hidden_dim=10):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, hidden_dim),
torch.nn.Linear(hidden_dim, 1)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesModel results
Training loss:
Predictions
Non-linear regression w/ ReLU
class lin_reg_relu(torch.nn.Module):
def __init__(self, X, hidden_dim=100):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, 1)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesModel results
Hidden dimensions
Non-linear regression w/ Tanh
class lin_reg_tanh(torch.nn.Module):
def __init__(self, X, hidden_dim=10):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, hidden_dim),
torch.nn.Tanh(),
torch.nn.Linear(hidden_dim, 1)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesTanh & hidden dimension
Three layers
class three_layers(torch.nn.Module):
def __init__(self, X, hidden_dim=100):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, 1)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesModel results
Five layers
class five_layers(torch.nn.Module):
def __init__(self, X, hidden_dim=100):
super().__init__()
self.n = X.shape[0]
self.p = X.shape[1]
self.model = torch.nn.Sequential(
torch.nn.Linear(self.p, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, hidden_dim),
torch.nn.ReLU(),
torch.nn.Linear(hidden_dim, 1)
)
def forward(self, X):
return self.model(X)
def fit(self, X, y, n=1000):
losses = []
opt = torch.optim.SGD(self.parameters(), lr=0.001, momentum=0.9)
for i in range(n):
loss = torch.nn.MSELoss()(self(X).squeeze(), y)
loss.backward()
opt.step()
opt.zero_grad()
losses.append(loss.item())
return lossesModel results
:::
Sta 663 - Spring 2023
pytorch - optim & nn Lecture 23 Dr. Colin Rundel