Merge pull request #2 from InCogNiTo124/dev

Fix mathsy bugs

layers.py

@@ -1,27 +1,22 @@
 import numpy as np
 
-class Linear():
-    def __init__(self, in_dimension, out_dimension):
-        self.W = np.random.randn(out_dimension, in_dimension)
+class Sigmoid():
+    def __init__(self):
         return
 
     def forward(self, X):
-        return self.W.dot(X)
+        return 1/(1+np.exp(-X))
 
-    def backward(self):
-        pass
+    def backward(self, delta):
+        return delta*(1-delta)
 
-    def __repr__(self):
-        return "Linear(W=\n{}\nb=\n{})".format(repr(self.W), repr(self.b))
 class Tanh():
+    def __init__(self):
+        return
 
-def tanh(x):
-    return np.tanh(x)
-
-def sigm(x):
-    return 1/(1+np.exp(-x))
+    def forward(self, X):
+        return np.tanh(X)
 
-def relu(x):
-    # pretty f***ing smart :D
-    return x * (x > 0)
+    def backward(self, delta):
+        return (1-delta) * (1+delta)
 

losses.py

@@ -10,13 +10,24 @@ def forward(self, y_true, y_pred):
     def backward(self, y_true, y_pred):
         pass
 
-class MeanSquaredError():
+class MSE(Loss):
     def __init__(self):
-        super().__init__(self)
+        super().__init__()
         return
 
-    def forward(self, y_true, y_pred):
-        return np.sum((y_true - y_pred)**2) / y_true.shape[1]
+    def forward(self, y_pred, y_true):
+        return np.mean((y_true - y_pred)**2)
 
     def backward(self, y_true, y_pred):
+        return y_true - y_pred
+
+class CE(Loss):
+    def __init__(self):
+        super().__init__()
         return
+
+    def forward(self, y_pred, y_true):
+        return -np.mean(y_true*np.log(y_pred) + (1-y_true)*np.log(1-y_pred))
+
+    def backward(self, y_pred, y_true):
+        return (y_pred - y_true) / (y_pred * (1- y_pred))

test.py

@@ -1,60 +1,38 @@
 import numpy as np
 import csv
+#from layers import Sigmoid
+from layers import Tanh as Sigmoid
+from losses import MSE
 LAYER_LIST = [2, 3, 1]
-LEARNING_RATE=1e-3
+LEARNING_RATE = 0.1
 np.random.seed(0)
 
-class MSE():
-    def __init__(self):
-        return
-
-    def forward(self, y_true, y_pred):
-        #print("y_true {} \t y_pred {}".format(y_true, y_pred))
-        return np.mean((y_true - y_pred)**2)
-
-
-    def backward(self, y_true, y_pred):
-        #print("y_true.shape", y_true.shape)
-        #print("y_pred.shape", y_pred.shape)
-        #print(y_pred)
-        #return np.mean(y_true - y_pred, axis=1, keepdims=True)
-        return y_true - y_pred
-        #return y_pred - y_true
 
 class Linear():
     def __init__(self, in_dim, out_dim):
-        self.W = np.random.randn(out_dim, in_dim)*0.2
-        self.b = np.random.randn(out_dim, 1)*0.2
+        self.W = np.random.randn(out_dim, in_dim)*0.5
+        self.b = np.random.randn(out_dim, 1)*0.5
         return
 
     def forward(self, X):
         return self.W.dot(X) + self.b
 
     def backward(self, delta):
-        #print(self.W.shape)
-        #print(delta.shape)
         return self.W.T.dot(delta)
 
-class Sigmoid():
-    def __init__(self):
-        return
-
-    def forward(self, X):
-        return 1/(1+np.exp(-X))
-
-    def backward(self, delta):
-        return delta*(1-delta)
-
 class Layer():
-    def __init__(self, in_dim, out_dim):
+    def __init__(self, in_dim, out_dim, activation=Sigmoid()):
+        assert in_dim > 0
+        assert out_dim > 0
+        assert type(in_dim) == type(0) # INT
+        assert type(out_dim) == type(0) # INT
         self.lin = Linear(in_dim, out_dim)
-        self.nlin = Sigmoid()
+        self.nlin = activation
         return
 
 def sgd(X, y_true, layer_list, loss, batch_size=4, epochs=1):
-    #print("X.shape {}\ny_true.shape {}".format(X.shape, y_true.shape))
     for epoch in range(1, epochs+1):
-        print("EPOCH: {}".format(epoch))
+        #print("EPOCH: {}".format(epoch))
         for i in range(X.shape[0]//batch_size):
             start = i*batch_size
             end = start + batch_size
@@ -65,36 +43,33 @@ def sgd(X, y_true, layer_list, loss, batch_size=4, epochs=1):
 
 def sgd_step(X, y, layer_list, loss):
     if layer_list == []:
-        #print("y_true {} \t y_pred {} \t loss {}".format(y, X, loss.forward(y, X)))
-        print(loss.forward(y, X))
+        print("y_true {} \n y_pred {} \n\t\t loss {}".format(y, X, loss.forward(y, X)))
         return loss.backward(X, y)
     else:
         layer = layer_list[0]
         z = layer.lin.forward(X)
         a = layer.nlin.forward(z)
-#        print("W.shape {} \t X.shape {} \t a.shape {}".format(layer.lin.W.shape, X.shape, a.shape))
         grad = sgd_step(a, y, layer_list[1:], loss)
-        # TODO: update
         n_grad = grad * layer.nlin.backward(a)
-        #print("n_grad", n_grad)
-#        print("grad.shape", n_grad.shape, end='\t')
-#        print("X.T.shape", X.T.shape)
-        #dw = np.mean(X.T * n_grad, axis=1, keepdims=True)
         dw = n_grad.dot(X.T)
-        #print(dw.shape)
-        #print(layer.lin.W.shape)
-        l_grad = layer.lin.backward(n_grad)
+        assert dw.shape == layer.lin.W.shape
+        l_grad = layer.lin.W.T @ n_grad
         layer.lin.W -= LEARNING_RATE*dw
-        layer.lin.b -= LEARNING_RATE*np.mean(n_grad, axis=1, keepdims=True)
         return l_grad
 
 if __name__ == '__main__':
     with open("dataset.csv", "r") as f:
         dataset = np.array(list(csv.reader(f, delimiter=",")), dtype=np.float64)
-    #print(dataset.shape)
-    #print(dataset[:10, :])
     layers = [Layer(i, o) for i, o in zip(LAYER_LIST, LAYER_LIST[1:])]
-    #print(layers)
     loss = MSE()
-    sgd(dataset[:, :2], dataset[:, 2:], layers, loss, batch_size=dataset.shape[0]//2, epochs=10000)
+    dataset[dataset[:, 2] == 0, 2] = -1
+    sgd(dataset[:, :2], dataset[:, 2:],
+        layers,
+        loss,
+        batch_size=32,
+        epochs=1000)
+    #TODO:
+    # - refactor layer class
+    # - add Tanh and relu activation
+    # - make python cli interface