diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..89d2399
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..1d3ce46
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 399e8ce..2a3d948 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -4,35 +4,47 @@
-
+
+
+
+
+
+
+
+
+
-
+ {
+ "associatedIndex": 7
+}
- {
+ "keyToString": {
+ "ModuleVcsDetector.initialDetectionPerformed": "true",
+ "Python.Unnamed.executor": "Run",
+ "Python.multilayer-perceptron.executor": "Run",
+ "RunOnceActivity.ShowReadmeOnStart": "true",
+ "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
+ "RunOnceActivity.git.unshallow": "true",
+ "git-widget-placeholder": "master",
+ "last_opened_file_path": "/home/arctichawk1/Desktop/Projects/Private/Classification-of-Image-Data-with-MLP-and-CNN"
}
-}]]>
+}
diff --git a/README.md b/README.md
index aaf77db..02280db 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# Classification-of-Image-Data-with-MLP-and-CNN
+# Classification of Image Data with MLP and CNN
This project implements both a multilayer perceptron and a convolutional neural network in Python;
the perceptron comprises an input layer, one or more hidden layers, and an output layer,
diff --git a/experiment-1-1.py b/experiment-1-1.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-1-2.py b/experiment-1-2.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-1-3.py b/experiment-1-3.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-2-leaky-relu.py b/experiment-2-leaky-relu.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-2-tanh.py b/experiment-2-tanh.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-3-l1.py b/experiment-3-l1.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-3-l2.py b/experiment-3-l2.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-4.py b/experiment-4.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-5.py b/experiment-5.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-6.py b/experiment-6.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-7.py b/experiment-7.py
new file mode 100644
index 0000000..e69de29
diff --git a/experiment-8.py b/experiment-8.py
new file mode 100644
index 0000000..e69de29
diff --git a/multilayer-perceptron.py b/multilayer-perceptron.py
index c6089b7..9bc2df8 100644
--- a/multilayer-perceptron.py
+++ b/multilayer-perceptron.py
@@ -1,8 +1,12 @@
import numpy as np
+import matplotlib.pyplot as plt
from torchvision import datasets
+import os
+
class MLP:
def __init__(self, input_size, hidden_size1, hidden_size2, output_size, weight_scale):
+ # initializes weights and biases for each layer
self.W1 = np.random.randn(input_size, hidden_size1) * weight_scale
self.b1 = np.zeros((1, hidden_size1))
self.W2 = np.random.randn(hidden_size1, hidden_size2) * weight_scale
@@ -11,31 +15,35 @@ class MLP:
self.b3 = np.zeros((1, output_size))
def forward(self, x):
- self.x = x
- self.z1 = x @ self.W1 + self.b1
- self.a1 = self.relu(self.z1)
- self.z2 = self.a1 @ self.W2 + self.b2
- self.a2 = self.relu(self.z2)
- self.z3 = self.a2 @ self.W3 + self.b3
- self.a3 = self.softmax(self.z3)
- return self.a3
+ # forwards pass through the network
+ self.x = x # input for backpropagation
+ self.z1 = x @ self.W1 + self.b1 # linear transformation for layer 1
+ self.a1 = self.relu(self.z1) # ReLU activation
+ self.z2 = self.a1 @ self.W2 + self.b2 # linear transformation for layer 2
+ self.a2 = self.relu(self.z2) # ReLU activation
+ self.z3 = self.a2 @ self.W3 + self.b3 # linear transformation for layer 3
+ self.a3 = self.softmax(self.z3) # applies softmax to get class probabilities
+ return self.a3 # output of the network
def backward(self, y, lr):
+ # backwards pass for weight updates using gradient descent
m = y.shape[0]
- y_one_hot = self.one_hot_encode(y, self.W3.shape[1])
+ y_one_hot = self.one_hot_encode(y, self.W3.shape[1]) # converts labels to one-hot encoding
- dz3 = self.a3 - y_one_hot
- dw3 = (self.a2.T @ dz3) / m
+ # computes gradients for each layer
+ dz3 = self.a3 - y_one_hot # gradient for output layer
+ dw3 = (self.a2.T @ dz3) / m
db3 = np.sum(dz3, axis=0, keepdims=True) / m
- dz2 = (dz3 @ self.W3.T) * self.relu_deriv(self.z2)
- dw2 = (self.a1.T @ dz2) / m
+ dz2 = (dz3 @ self.W3.T) * self.relu_deriv(self.z2) # gradient for layer 2
+ dw2 = (self.a1.T @ dz2) / m
db2 = np.sum(dz2, axis=0, keepdims=True) / m
- dz1 = (dz2 @ self.W2.T) * self.relu_deriv(self.z1)
- dw1 = (self.x.T @ dz1) / m
+ dz1 = (dz2 @ self.W2.T) * self.relu_deriv(self.z1) # gradient for layer 1
+ dw1 = (self.x.T @ dz1) / m
db1 = np.sum(dz1, axis=0, keepdims=True) / m
+ # updates weights and biases using gradient descent
self.W3 -= lr * dw3
self.b3 -= lr * db3
self.W2 -= lr * dw2
@@ -45,32 +53,41 @@ class MLP:
@staticmethod
def relu(x):
+ # ReLU activation
return np.maximum(0, x)
@staticmethod
def relu_deriv(x):
+ # derivation of ReLU activation for backpropagation
return (x > 0).astype(float)
@staticmethod
def softmax(x):
- e_x = np.exp(x - np.max(x, axis=1, keepdims=True))
- return e_x / np.sum(e_x, axis=1, keepdims=True)
+ # softmax function normalizes outputs to probabilities
+ e_x = np.exp(x - np.max(x, axis=1, keepdims=True)) # exponentiates inputs
+ return e_x / np.sum(e_x, axis=1, keepdims=True) # normalizes to get probabilities
@staticmethod
def one_hot_encode(y, num_classes):
+ # converts labels to one-hot encoded format
return np.eye(num_classes)[y]
@staticmethod
def cross_entropy_loss(y, y_hat):
+ # computes cross-entropy loss between true labels and predicted probabilities
+ m = y.shape[0]
m = y.shape[0]
eps = 1e-12
y_hat_clipped = np.clip(y_hat, eps, 1. - eps)
log_probs = -np.log(y_hat_clipped[np.arange(m), y])
return np.mean(log_probs)
- def train_model(self, x_train, y_train, x_val, y_val, lr, epochs, batch_size):
+ def fit(self, x_train, y_train, x_val, y_val, lr, epochs, batch_size):
+ train_losses = []
+ val_accuracies = []
+
for epoch in range(1, epochs + 1):
- perm = np.random.permutation(x_train.shape[0])
+ perm = np.random.permutation(x_train.shape[0]) # Shuffle the training data
x_train_shuffled, y_train_shuffled = x_train[perm], y_train[perm]
epoch_loss = 0.0
@@ -78,56 +95,86 @@ class MLP:
for i in range(num_batches):
start = i * batch_size
- end = start + batch_size
- x_batch = x_train_shuffled[start:end]
- y_batch = y_train_shuffled[start:end]
+ end = start + batch_size
+ x_batch = x_train_shuffled[start:end] # batch of inputs
+ y_batch = y_train_shuffled[start:end] # batch of labels
+ # Forward pass, backward pass, and weight update
self.forward(x_batch)
self.backward(y_batch, lr)
- epoch_loss += self.cross_entropy_loss(y_batch, self.a3)
+ epoch_loss += self.cross_entropy_loss(y_batch, self.a3) # updating the epoch loss
- epoch_loss /= num_batches
+ epoch_loss /= num_batches # average loss is defined
+ train_losses.append(epoch_loss)
val_pred = self.predict(x_val)
- val_acc = np.mean(val_pred == y_val)
+ val_acc = np.mean(val_pred == y_val)
+ val_accuracies.append(val_acc) \
print(f"Epoch {epoch:02d} | Training Loss: {epoch_loss:.4f} | Value Accuracy: {val_acc:.4f}")
- return val_acc
+ self.plot_graph(train_losses, val_accuracies)
+ return val_accuracies[-1]
- def predict(self, x):
- probs = self.forward(x)
- return np.argmax(probs, axis=1)
+ def plot_graph(self, train_losses, val_accuracies):
+ if not os.path.exists('results'):
+ os.makedirs('results')
+ fig, ax1 = plt.subplots()
+ ax1.set_xlabel('Epochs')
+ ax1.set_ylabel('Training Loss', color='tab:blue')
+ ax1.plot(range(1, len(train_losses) + 1), train_losses, color='tab:blue', label='Training Loss')
+ ax1.tick_params(axis='y', labelcolor='tab:blue')
+
+ ax2 = ax1.twinx()
+ ax2.set_ylabel('Validation Accuracy', color='tab:orange')
+ ax2.plot(range(1, len(val_accuracies) + 1), val_accuracies, color='tab:orange', label='Validation Accuracy')
+ ax2.tick_params(axis='y', labelcolor='tab:orange')
+
+ plt.title('Training Loss and Validation Accuracy over Epochs')
+
+ result_path = 'results/MLP-output.png'
+ fig.savefig(result_path)
+ print(f"Graph saved to: {result_path}")
+
+ def predict(self, x): # predicts class labels for the input data
+ probs = self.forward(x) # forwards pass to get probabilities
+ return np.argmax(probs, axis=1) # returns the class with highest probability
+
+# acquiring the FashionMNIST dataset
train_set = datasets.FashionMNIST(root='.', train=True, download=True)
-test_set = datasets.FashionMNIST(root='.', train=False, download=True)
+test_set = datasets.FashionMNIST(root='.', train=False, download=True)
+# preprocessing the data by flattening images and normalizing them.
x_train = train_set.data.numpy().reshape(-1, 28 * 28).astype(np.float32) / 255.0
y_train = train_set.targets.numpy()
-x_test = test_set.data.numpy().reshape(-1, 28 * 28).astype(np.float32) / 255.0
-y_test = test_set.targets.numpy()
+x_test = test_set.data.numpy().reshape(-1, 28 * 28).astype(np.float32) / 255.0
+y_test = test_set.targets.numpy()
+# MLP Initialization
mlp = MLP(
- input_size = 28 * 28,
- hidden_size1= 128,
- hidden_size2= 64,
- output_size = 10,
- weight_scale= 1e-2
+ input_size=28 * 28,
+ hidden_size1=128,
+ hidden_size2=64,
+ output_size=10,
+ weight_scale=1e-2
)
-mlp.train_model(
- x_train = x_train,
- y_train = y_train,
- x_val = x_test,
- y_val = y_test,
- lr = 1e-2,
- epochs = 10,
+# trains the model
+mlp.fit(
+ x_train=x_train,
+ y_train=y_train,
+ x_val=x_test,
+ y_val=y_test,
+ lr=1e-2,
+ epochs=10,
batch_size=128
)
+# tests the model
test_pred = mlp.predict(x_test)
-test_acc = np.mean(test_pred == y_test)
-print(f"\nFinal test accuracy: {test_acc:.4f}")
+test_acc = np.mean(test_pred == y_test)
+print(f"\nFinal test accuracy: {test_acc:.4f}")
\ No newline at end of file
diff --git a/results/MLP-output.png b/results/MLP-output.png
new file mode 100644
index 0000000..7e9efc1
Binary files /dev/null and b/results/MLP-output.png differ