Backpropagation Algorithm Implementation for a Multi Layer Perceptron

.vscode/settings.json

@@ -17,7 +17,6 @@
   "files.autoSave": "afterDelay",
   "screencastMode.onlyKeyboardShortcuts": true,
   "terminal.integrated.fontSize": 18,
-  "workbench.activityBar.visible": true,
   "workbench.colorTheme": "Visual Studio Dark",
   "workbench.fontAliasing": "antialiased",
   "workbench.statusBar.visible": true,

AND-Gate/main.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+class Perceptron:
+    def __init__(self, inputs, bias=1.0):
+        # Initialize weights randomly between -1 and 1
+        self.weights = (np.random.rand(inputs + 1) * 2) - 1
+        self.bias = bias
+
+    def run(self, x):
+        # Append the bias to the input list
+        x_with_bias = np.append(x, self.bias)
+        # Calculate weighted sum
+        x_sum = np.dot(x_with_bias, self.weights)
+        # Apply sigmoid activation function
+        return self.sigmoid(x_sum)
+
+    def set_weights(self, w_init):
+        # Manually set weights for specific logic gates
+        self.weights = np.array(w_init)
+
+    def sigmoid(self, x):
+        # Activation function to squash output between 0 and 1
+        return 1 / (1 + np.exp(-x))
+
+# --- TESTING AND GATE ---
+neuron = Perceptron(inputs=2)
+neuron.set_weights([10, 10, -15]) # AND Gate weights
+
+print("Gate (AND):")
+print(f"0 0 = {neuron.run([0, 0]):.10f}")
+print(f"0 1 = {neuron.run([0, 1]):.10f}")
+print(f"1 0 = {neuron.run([1, 0]):.10f}")
+print(f"1 1 = {neuron.run([1, 1]):.10f}")
+
+print("-" * 20)

Backpropagation/main.py

@@ -0,0 +1,84 @@
+import numpy as np
+
+class Perceptron:
+    def __init__(self, inputs, bias=1.0):
+        self.weights = (np.random.rand(inputs + 1) * 2) - 1
+        self.bias = bias
+
+    def run(self, x):
+        x_with_bias = np.append(x, self.bias)
+        x_sum = np.dot(x_with_bias, self.weights)
+        return self.sigmoid(x_sum)
+
+    def set_weights(self, w_init):
+        self.weights = np.array(w_init)
+
+    def sigmoid(self, x):
+        return 1 / (1 + np.exp(-x))
+
+class MultiLayerPerceptron:
+    def __init__(self, layers, bias=1.0, eta=0.5):
+        self.layers = np.array(layers, dtype=object)
+        self.bias = bias
+        self.eta = eta
+        self.network = []
+        self.values = []
+        self.d = []
+
+        for i in range(len(self.layers)):
+            self.values.append([0.0 for j in range(self.layers[i])])
+            self.d.append([0.0 for j in range(self.layers[i])])
+            if i > 0:
+                layer = [Perceptron(inputs=self.layers[i-1], bias=self.bias) for j in range(self.layers[i])]
+                self.network.append(layer)
+
+        self.network = np.array([np.array(x) for x in self.network], dtype=object)
+        self.values = np.array([np.array(x) for x in self.values], dtype=object)
+        self.d = np.array([np.array(x) for x in self.d], dtype=object)
+
+    def run(self, x):
+        self.values[0] = x
+        for i in range(1, len(self.network) + 1):
+            for j in range(self.layers[i]):
+                self.values[i][j] = self.network[i-1][j].run(self.values[i-1])
+        return self.values[-1]
+
+    def bp(self, x, y):
+        outputs = self.run(x)
+        error = (y - outputs)
+
+        # Calculate output layer error terms (deltas)
+        self.d[-1] = outputs * (1 - outputs) * error
+
+        # Calculate error terms for hidden layers
+        for i in reversed(range(1, len(self.network))):
+            for h in range(len(self.network[i-1])):
+                fwd_error = 0.0
+                for k in range(self.layers[i+1]):
+                    fwd_error += self.network[i][k].weights[h] * self.d[i+1][k]
+                self.d[i][h] = self.values[i][h] * (1 - self.values[i][h]) * fwd_error
+
+        # Update weights
+        for i in range(1, len(self.network) + 1):
+            for j in range(self.layers[i]):
+                for k in range(self.layers[i-1] + 1):
+                    if k == self.layers[i-1]:
+                        delta = self.eta * self.d[i][j] * self.bias
+                    else:
+                        delta = self.eta * self.d[i][j] * self.values[i-1][k]
+                    self.network[i-1][j].weights[k] += delta
+        return sum(error**2)
+
+# Training
+mlp = MultiLayerPerceptron(layers=[2, 2, 1])
+print("Training Neural Network as an XOR Gate...\n")
+for i in range(5000):
+    mse = (mlp.bp([0,0],[0]) + mlp.bp([0,1],[1]) + mlp.bp([1,0],[1]) + mlp.bp([1,1],[0])) / 4
+    if i % 500 == 0:
+        print(f"MSE: {mse:.6f}")
+
+print("\nFinal XOR Results:")
+print(f"0 0 = {mlp.run([0,0])[0]:.4f}")
+print(f"0 1 = {mlp.run([0,1])[0]:.4f}")
+print(f"1 0 = {mlp.run([1,0])[0]:.4f}")
+print(f"1 1 = {mlp.run([1,1])[0]:.4f}")

Multi-Layer-Perceptron/main.py

@@ -0,0 +1,136 @@
+import numpy as np
+
+
+class Perceptron:
+    def __init__(self, inputs, bias=1.0):
+        # Initialize random weights (including one weight for the bias)
+        self.weights = (np.random.rand(inputs + 1) * 2) - 1
+        self.bias = bias
+
+    def run(self, x):
+        # Add the bias input
+        x_with_bias = np.append(x, self.bias)
+
+        # Calculate weighted sum
+        x_sum = np.dot(x_with_bias, self.weights)
+
+        # Apply sigmoid activation
+        return self.sigmoid(x_sum)
+
+    def set_weights(self, w_init):
+        self.weights = np.array(w_init)
+
+    def sigmoid(self, x):
+        return 1 / (1 + np.exp(-x))
+
+
+class MultiLayerPerceptron:
+    def __init__(self, layers, bias=1.0):
+        self.layers = np.array(layers, dtype=object)
+        self.bias = bias
+
+        # Stores all neurons in the network
+        self.network = []
+
+        # Stores the output value of every neuron
+        self.values = []
+
+        # Build the network
+        for i in range(len(self.layers)):
+            # Allocate memory for each layer's outputs
+            self.values.append([0.0 for _ in range(self.layers[i])])
+
+            # Skip the input layer (it contains no neurons)
+            if i > 0:
+                layer = [
+                    Perceptron(inputs=self.layers[i - 1], bias=self.bias)
+                    for _ in range(self.layers[i])
+                ]
+                self.network.append(layer)
+
+        self.network = np.array(
+            [np.array(layer) for layer in self.network],
+            dtype=object
+        )
+
+        self.values = np.array(
+            [np.array(layer) for layer in self.values],
+            dtype=object
+        )
+
+    def set_weights(self, weights):
+        """
+        Assign manually configured weights to every neuron.
+
+        Network Structure:
+        Input(2) -> Hidden(2) -> Output(1)
+
+        Hidden Layer
+            Neuron 1 : [-10, -10, 15]
+            Neuron 2 : [15, 15, -10]
+
+        Output Layer
+            Neuron 1 : [10, 10, -15]
+        """
+
+        for layer_index in range(len(weights)):
+            for neuron_index in range(len(weights[layer_index])):
+                self.network[layer_index][neuron_index].set_weights(
+                    weights[layer_index][neuron_index]
+                )
+
+    def print_weights(self):
+        print("Neural Network Weights\n")
+
+        for layer in range(1, len(self.network) + 1):
+            for neuron in range(len(self.network[layer - 1])):
+                print(
+                    f"Layer {layer}  "
+                    f"Neuron {neuron} : "
+                    f"{self.network[layer - 1][neuron].weights}"
+                )
+
+        print()
+
+    def run(self, x):
+        # Input layer values
+        self.values[0] = x
+
+        # Forward propagation
+        for layer in range(1, len(self.network) + 1):
+            for neuron in range(self.layers[layer]):
+                self.values[layer][neuron] = (
+                    self.network[layer - 1][neuron].run(
+                        self.values[layer - 1]
+                    )
+                )
+
+        return self.values[-1]
+
+
+# ----------------------------------------------------
+# XOR Demonstration
+# ----------------------------------------------------
+
+# Create a 2-2-1 Multi-Layer Perceptron
+mlp = MultiLayerPerceptron(layers=[2, 2, 1])
+
+# Manually assign weights that solve XOR
+mlp.set_weights([
+    [
+        [-10, -10, 15],
+        [15, 15, -10]
+    ],
+    [
+        [10, 10, -15]
+    ]
+])
+
+mlp.print_weights()
+
+print("Multi-Layer Perceptron (XOR Gate)\n")
+
+print(f"0 XOR 0 = {mlp.run([0, 0])[0]:.10f}")
+print(f"0 XOR 1 = {mlp.run([0, 1])[0]:.10f}")
+print(f"1 XOR 0 = {mlp.run([1, 0])[0]:.10f}")
+print(f"1 XOR 1 = {mlp.run([1, 1])[0]:.10f}")

OR-Gate/main.py

@@ -0,0 +1,47 @@
+import numpy as np
+
+class Perceptron:
+    def __init__(self, inputs, bias=1.0):
+        # Initialize weights randomly between -1 and 1
+        self.weights = (np.random.rand(inputs + 1) * 2) - 1
+        self.bias = bias
+
+    def run(self, x):
+        # Append the bias to the input list
+        x_with_bias = np.append(x, self.bias)
+        # Calculate weighted sum
+        x_sum = np.dot(x_with_bias, self.weights)
+        # Apply sigmoid activation function
+        return self.sigmoid(x_sum)
+
+    def set_weights(self, w_init):
+        # Manually set weights for specific logic gates
+        self.weights = np.array(w_init)
+
+    def sigmoid(self, x):
+        # Activation function to squash output between 0 and 1
+        return 1 / (1 + np.exp(-x))
+
+# --- TESTING AND GATE ---
+neuron = Perceptron(inputs=2)
+neuron.set_weights([10, 10, -15]) # AND Gate weights
+
+print("Gate (AND):")
+print(f"0 0 = {neuron.run([0, 0]):.10f}")
+print(f"0 1 = {neuron.run([0, 1]):.10f}")
+print(f"1 0 = {neuron.run([1, 0]):.10f}")
+print(f"1 1 = {neuron.run([1, 1]):.10f}")
+
+print("-" * 20)
+
+# --- TESTING OR GATE ---
+neuron = Perceptron(inputs=2)
+neuron.set_weights([15, 15, -10]) # OR Gate weights
+
+print("Gate (OR):")
+print(f"0 0 = {neuron.run([0, 0]):.10f}")
+print(f"0 1 = {neuron.run([0, 1]):.10f}")
+print(f"1 0 = {neuron.run([1, 0]):.10f}")
+print(f"1 1 = {neuron.run([1, 1]):.10f}")
+
+