{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import imageio.v3 as imageio\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from dataclasses import dataclass\n",
    "from typing import Literal\n",
    "import seaborn as sns\n",
    "from matplotlib import cm\n",
    "from sklearn.datasets import make_moons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab21"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cost functions\n",
    "\n",
    "def cost_cross_entropy(y, yhat):\n",
    "    eps = 1e-8\n",
    "    return -(yhat * np.log(y+eps) + (1-yhat) * np.log(1-y+eps))\n",
    "\n",
    "def cost_cross_entropy_derivative(y, yhat):\n",
    "    eps = 1e-8\n",
    "    return (1-yhat)/(1-y+eps) - yhat/(y+eps)\n",
    "\n",
    "\n",
    "def cost_mse(y, yhat):\n",
    "    return np.sum((yhat - y)**2) / (2 * y.shape[0])\n",
    "\n",
    "def cost_mse_derivative(y, yhat):\n",
    "    return -2 * np.sum((yhat - y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward propagation & Backward propagation variants\n",
    "\n",
    "def f(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def dfdz(z):\n",
    "    return f(z) * (1 - f(z))\n",
    "\n",
    "def forward(x, w, b):\n",
    "    return f(np.matmul(x,w)+b)\n",
    "\n",
    "## Option one: use the given derivatives (clearer)\n",
    "def backward1(x, w, b, y, yhat, alpha, dJdy):\n",
    "    z = np.dot(x, w)\n",
    "    delta = dJdy(y, yhat) * dfdz(z)\n",
    "\n",
    "    dw = np.reshape(delta*x, (x.shape[0], -1))\n",
    "    db = delta\n",
    "\n",
    "    w -= alpha*dw\n",
    "    b -= alpha*db\n",
    "\n",
    "    return w, b\n",
    "\n",
    "## Option two: use the explicit calculted form, uses cross entropy (faster)\n",
    "def backward2(x, w, b, y, yhat, alpha):  # w not needed for this form\n",
    "    delta = ( y*(1-yhat) - yhat*(1-y) )\n",
    "\n",
    "    dw = np.reshape(delta*x, (2, -1))\n",
    "    db = delta\n",
    "\n",
    "    w -= alpha*dw\n",
    "    b -= alpha*db\n",
    "\n",
    "    return w, b\n",
    "\n",
    "def backward3(X, w, b, y, yhat, alpha):\n",
    "    delta = ( y*(1-yhat) - yhat*(1-y) )\n",
    "\n",
    "    dw = np.reshape(np.mean(delta * X, axis=0), (2, -1))\n",
    "    db = np.mean(delta)\n",
    "\n",
    "    w -= alpha*dw\n",
    "    b -= alpha*db\n",
    "\n",
    "    return w, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training function\n",
    "\n",
    "def train(X, yhat, n_epoch, alpha, J=cost_cross_entropy, dJdy=cost_cross_entropy_derivative, rounding=0):\n",
    "    costs = np.zeros(shape=(n_epoch,))\n",
    "\n",
    "    np.random.seed(1)\n",
    "    w = np.random.normal(0, 0.01, size=(X.shape[1],1))\n",
    "    b = np.random.normal(0, 0.01)\n",
    "\n",
    "    for epoch in range(n_epoch):\n",
    "\n",
    "        for i, x in enumerate(X):\n",
    "            y = forward(x, w, b)  # prediction for one sample\n",
    "            w, b = backward1(x, w, b, y, yhat[i], alpha, dJdy)  # take step\n",
    "\n",
    "        # Calculate total cost after epoch\n",
    "        predictions = forward(X, w, b)  # predictions for entire set\n",
    "        costs[epoch] = np.mean(J(predictions, yhat))  # mean cost per sample\n",
    "\n",
    "        # report progress\n",
    "        if ((epoch % 10) == 0) or (epoch == (n_epoch - 1)):\n",
    "            accuracy = np.mean((predictions.round(rounding) - yhat) < 1/(10**rounding))  # current accuracy on entire set\n",
    "            print('Training accuracy after epoch {}: {:.4%}'.format(epoch, accuracy))\n",
    "\n",
    "    return w, b, costs\n",
    "\n",
    "def train_batched(X, yhat, n_epoch, alpha, rounding=0):\n",
    "    costs = np.zeros(shape=(n_epoch,))\n",
    "\n",
    "    J = cost_cross_entropy\n",
    "\n",
    "    np.random.seed(1)\n",
    "    w = np.random.normal(0, 0.01, size=(X.shape[1],1))\n",
    "    b = np.random.normal(0, 0.01)\n",
    "\n",
    "    for epoch in range(n_epoch):\n",
    "        y = forward(X, w, b)\n",
    "        w, b = backward3(X, w, b, y, yhat, alpha)\n",
    "\n",
    "        # Calculate total cost after epoch\n",
    "        predictions = forward(X, w, b)  # predictions for entire set\n",
    "        costs[epoch] = np.mean(J(predictions, yhat))  # mean cost per sample\n",
    "\n",
    "        # report progress\n",
    "        if ((epoch % 10) == 0) or (epoch == (n_epoch - 1)):\n",
    "            accuracy = np.mean((predictions.round(rounding) - yhat) < 1/(10**rounding))  # current accuracy on entire set\n",
    "            print('Training accuracy after epoch {}: {:.4%}'.format(epoch, accuracy))\n",
    "\n",
    "    return w, b, costs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Studijuokite (Lab21) pateiktą interaktyvų programos kodą.\n",
    "\n",
    "# [x] Done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after epoch 0: 75.0000%\n",
      "Training accuracy after epoch 10: 100.0000%\n",
      "Training accuracy after epoch 20: 100.0000%\n",
      "Training accuracy after epoch 30: 50.0000%\n",
      "Training accuracy after epoch 40: 50.0000%\n",
      "Training accuracy after epoch 50: 50.0000%\n",
      "Training accuracy after epoch 60: 50.0000%\n",
      "Training accuracy after epoch 70: 50.0000%\n",
      "Training accuracy after epoch 80: 50.0000%\n",
      "Training accuracy after epoch 90: 50.0000%\n",
      "Training accuracy after epoch 100: 50.0000%\n",
      "Training accuracy after epoch 110: 50.0000%\n",
      "Training accuracy after epoch 120: 50.0000%\n",
      "Training accuracy after epoch 130: 50.0000%\n",
      "Training accuracy after epoch 140: 50.0000%\n",
      "Training accuracy after epoch 150: 50.0000%\n",
      "Training accuracy after epoch 160: 50.0000%\n",
      "Training accuracy after epoch 170: 50.0000%\n",
      "Training accuracy after epoch 180: 50.0000%\n",
      "Training accuracy after epoch 190: 50.0000%\n",
      "Training accuracy after epoch 199: 50.0000%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 31732 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2. Pakeiskite parametrus - mokymosi greiti, epochų skaičių.\n",
    "\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "yhat = np.array([[0],[1],[1],[1]])\n",
    "\n",
    "w, b, costs = train(X, yhat, 200, 1)\n",
    "plt.plot(costs);\n",
    "plt.title('Vidutinės nuostolių funkcijos reikšmės kiekvienai epochai');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after epoch 0: 75.0000%\n",
      "Training accuracy after epoch 10: 100.0000%\n",
      "Training accuracy after epoch 20: 100.0000%\n",
      "Training accuracy after epoch 30: 100.0000%\n",
      "Training accuracy after epoch 40: 100.0000%\n",
      "Training accuracy after epoch 50: 100.0000%\n",
      "Training accuracy after epoch 60: 100.0000%\n",
      "Training accuracy after epoch 70: 100.0000%\n",
      "Training accuracy after epoch 80: 100.0000%\n",
      "Training accuracy after epoch 90: 100.0000%\n",
      "Training accuracy after epoch 100: 100.0000%\n",
      "Training accuracy after epoch 110: 100.0000%\n",
      "Training accuracy after epoch 120: 100.0000%\n",
      "Training accuracy after epoch 130: 100.0000%\n",
      "Training accuracy after epoch 140: 100.0000%\n",
      "Training accuracy after epoch 150: 100.0000%\n",
      "Training accuracy after epoch 160: 100.0000%\n",
      "Training accuracy after epoch 170: 100.0000%\n",
      "Training accuracy after epoch 180: 100.0000%\n",
      "Training accuracy after epoch 190: 100.0000%\n",
      "Training accuracy after epoch 199: 100.0000%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 29920 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 3. Pakeiskite nuostolių funkciją į MSE.\n",
    "\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "yhat = np.array([[0],[1],[1],[1]])\n",
    "\n",
    "w, b, costs = train(X, yhat, 200, 1, cost_mse, cost_mse_derivative)\n",
    "plt.plot(costs);\n",
    "plt.title('Vidutinės nuostolių funkcijos reikšmės kiekvienai epochai');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after epoch 0: 55.8904%\n",
      "Training accuracy after epoch 10: 49.5890%\n",
      "Training accuracy after epoch 20: 48.2192%\n",
      "Training accuracy after epoch 30: 48.4932%\n",
      "Training accuracy after epoch 40: 49.3151%\n",
      "Training accuracy after epoch 50: 49.8630%\n",
      "Training accuracy after epoch 60: 50.1370%\n",
      "Training accuracy after epoch 70: 50.6849%\n",
      "Training accuracy after epoch 80: 51.2329%\n",
      "Training accuracy after epoch 90: 52.3288%\n",
      "Training accuracy after epoch 100: 52.8767%\n",
      "Training accuracy after epoch 110: 53.1507%\n",
      "Training accuracy after epoch 120: 53.1507%\n",
      "Training accuracy after epoch 130: 53.1507%\n",
      "Training accuracy after epoch 140: 53.4247%\n",
      "Training accuracy after epoch 150: 53.9726%\n",
      "Training accuracy after epoch 160: 54.7945%\n",
      "Training accuracy after epoch 170: 55.3425%\n",
      "Training accuracy after epoch 180: 56.1644%\n",
      "Training accuracy after epoch 190: 56.4384%\n",
      "Training accuracy after epoch 200: 56.9863%\n",
      "Training accuracy after epoch 210: 57.2603%\n",
      "Training accuracy after epoch 220: 57.2603%\n",
      "Training accuracy after epoch 230: 57.5342%\n",
      "Training accuracy after epoch 240: 57.5342%\n",
      "Training accuracy after epoch 250: 57.5342%\n",
      "Training accuracy after epoch 260: 57.8082%\n",
      "Training accuracy after epoch 270: 58.0822%\n",
      "Training accuracy after epoch 280: 58.0822%\n",
      "Training accuracy after epoch 290: 58.6301%\n",
      "Training accuracy after epoch 300: 58.6301%\n",
      "Training accuracy after epoch 310: 58.9041%\n",
      "Training accuracy after epoch 320: 59.1781%\n",
      "Training accuracy after epoch 330: 59.1781%\n",
      "Training accuracy after epoch 340: 59.7260%\n",
      "Training accuracy after epoch 350: 60.0000%\n",
      "Training accuracy after epoch 360: 60.0000%\n",
      "Training accuracy after epoch 370: 60.2740%\n",
      "Training accuracy after epoch 380: 60.2740%\n",
      "Training accuracy after epoch 390: 60.2740%\n",
      "Training accuracy after epoch 400: 60.2740%\n",
      "Training accuracy after epoch 410: 60.0000%\n",
      "Training accuracy after epoch 420: 60.2740%\n",
      "Training accuracy after epoch 430: 60.5479%\n",
      "Training accuracy after epoch 440: 60.8219%\n",
      "Training accuracy after epoch 450: 61.3699%\n",
      "Training accuracy after epoch 460: 61.6438%\n",
      "Training accuracy after epoch 470: 61.9178%\n",
      "Training accuracy after epoch 480: 61.9178%\n",
      "Training accuracy after epoch 490: 61.6438%\n",
      "Training accuracy after epoch 499: 61.9178%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 38220 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4. Raskite realaus pasaulio duomenų rinkinį ir pritaikykite šį modelį.\n",
    "\n",
    "def map_value(x, from_min, from_max, to_min, to_max):\n",
    "    return (x - from_min) / (from_max - from_min) * (to_max - to_min) + to_min\n",
    "\n",
    "def normalize_column(column):\n",
    "    min_value = min(column)\n",
    "    max_value = max(column)\n",
    "\n",
    "    return map(lambda x: map_value(float(x), min_value, max_value, 0, 1), column)\n",
    "\n",
    "def prepare_number_list(numbers):\n",
    "    return np.array(list(map(lambda value: [float(value)], numbers)))\n",
    "\n",
    "df = pd.read_csv('assets/Ice Cream Sales - temperatures.csv')\n",
    "X = prepare_number_list(normalize_column(df['Temperature'].values))\n",
    "yhat = prepare_number_list(normalize_column(df['Ice Cream Profits'].values))\n",
    "\n",
    "w, b, costs = train(X, yhat, 500, 0.003, cost_mse, cost_mse_derivative, rounding=3)\n",
    "plt.plot(costs);\n",
    "plt.title('Vidutinės nuostolių funkcijos reikšmės kiekvienai epochai');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy after epoch 0: 75.0000%\n",
      "Training accuracy after epoch 10: 75.0000%\n",
      "Training accuracy after epoch 20: 100.0000%\n",
      "Training accuracy after epoch 30: 100.0000%\n",
      "Training accuracy after epoch 40: 100.0000%\n",
      "Training accuracy after epoch 50: 100.0000%\n",
      "Training accuracy after epoch 60: 100.0000%\n",
      "Training accuracy after epoch 70: 100.0000%\n",
      "Training accuracy after epoch 80: 100.0000%\n",
      "Training accuracy after epoch 90: 100.0000%\n",
      "Training accuracy after epoch 100: 100.0000%\n",
      "Training accuracy after epoch 110: 100.0000%\n",
      "Training accuracy after epoch 120: 100.0000%\n",
      "Training accuracy after epoch 130: 100.0000%\n",
      "Training accuracy after epoch 140: 100.0000%\n",
      "Training accuracy after epoch 150: 100.0000%\n",
      "Training accuracy after epoch 160: 100.0000%\n",
      "Training accuracy after epoch 170: 100.0000%\n",
      "Training accuracy after epoch 180: 100.0000%\n",
      "Training accuracy after epoch 190: 100.0000%\n",
      "Training accuracy after epoch 199: 100.0000%\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 30796 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 5. Perdarykite algoritmą, kad gradientas būtų skaičiuojamas ne atskiroms  X\n",
    "# reikšmėms, o reikšmių rinkiniams (batches).\n",
    "\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "yhat = np.array([[0],[1],[1],[1]])\n",
    "\n",
    "w, b, costs = train_batched(X, yhat, 200, 1)\n",
    "plt.plot(costs);\n",
    "plt.title('Vidutinės nuostolių funkcijos reikšmės kiekvienai epochai');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Studijuokite ir eksperimentuokite naudojant https://playground.tensorflow.org\n",
    "\n",
    "# [x] Done"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualization functions\n",
    "\n",
    "def plot_graph(X, y, pred_line=None, losses=None):\n",
    "\n",
    "    plots = 2 if losses!=None else 1\n",
    "\n",
    "    fig = plt.figure(figsize=(8 * plots, 6))\n",
    "\n",
    "    ax1 = fig.add_subplot(1, plots, 1)\n",
    "    ax1.scatter(X, y, alpha=0.8)                                # Plot the original set of datapoints\n",
    "\n",
    "    if(pred_line != None):\n",
    "        x_line, y_line = pred_line['x_line'], pred_line['y_line']\n",
    "        ax1.plot(x_line, y_line, linewidth=2, markersize=12, color='red', alpha=0.8)      # Plot the randomly generated line\n",
    "        ax1.set_title('Predicted Line on set of Datapoints')\n",
    "    else:\n",
    "        ax1.set_title('Plot of Datapoints generated')\n",
    "\n",
    "    ax1.set_xlabel('x')\n",
    "    ax1.set_ylabel('y')\n",
    "\n",
    "    if(losses!=None):\n",
    "        ax2 = fig.add_subplot(1, plots, 2)\n",
    "        ax2.plot(np.arange(len(losses)), losses, marker='o')\n",
    "\n",
    "        ax2.set_xlabel('Epoch')\n",
    "        ax2.set_ylabel('Loss')\n",
    "        ax2.set_title('Loss')\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "def plot_pred_line(X, y, w, b,losses=None):\n",
    "    # Generate a set of datapoints on x for creating a line.\n",
    "    # We shall consider the range of X_train for generating the line so that the line superposes the datapoints.\n",
    "    x_line = np.linspace(np.min(X), np.max(X), 10)\n",
    "\n",
    "    # Calculate the corresponding y with the parameter values of m & b\n",
    "    y_line = w[0] * x_line + b\n",
    "\n",
    "    plot_graph(X[:, 0], y, pred_line={'x_line': x_line, 'y_line':y_line}, losses=losses)\n",
    "\n",
    "    return\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main\n",
    "\n",
    "def forward_prop(X, w, b):\n",
    "    #y_pred = w * X + b\n",
    "    y_pred = np.reshape(np.sum(w*X,1),(X.shape[0],1)) + b\n",
    "    return y_pred\n",
    "\n",
    "def compute_loss(y, y_pred):\n",
    "    loss = np.mean((y_pred - y)**2)\n",
    "\n",
    "    return loss\n",
    "\n",
    "def grad_desc(w, b, X_train, y_train, y_pred):\n",
    "    dw = np.mean(2*(y_pred - y_train) * X_train)\n",
    "    db = np.mean(2*(y_pred - y_train))\n",
    "\n",
    "    return dw, db\n",
    "\n",
    "def back_prop(X_train, y_train, y_pred, w, b, l_r):\n",
    "    dw, db = grad_desc(w, b, X_train, y_train, y_pred)\n",
    "\n",
    "    w -= l_r * dw\n",
    "    b -= l_r * db\n",
    "\n",
    "    return w, b\n",
    "\n",
    "def train_and_show(X, y, epochs, l_r, show_every=10):\n",
    "    m = np.random.normal(scale=10)\n",
    "    b = np.random.normal(scale=10)\n",
    "    w = np.random.normal(scale=10, size=(X.shape[1],))\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "\n",
    "    losses = []\n",
    "\n",
    "    for i in range(epochs):\n",
    "        y_pred = forward_prop(X_train, w, b)\n",
    "\n",
    "        #print(y_pred)\n",
    "\n",
    "        loss = compute_loss(y_train, y_pred)\n",
    "        losses.append(loss)\n",
    "\n",
    "        w, b = back_prop(X_train, y_train, y_pred, w, b, l_r)\n",
    "\n",
    "        if (i%show_every==0):\n",
    "            print('Epoch: ', i)\n",
    "            print('Loss = ', loss)\n",
    "            plot_pred_line(X_train, y_train, w, b, losses)\n",
    "\n",
    "    del losses[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Studijuokite (Lab22) pateiktą interaktyvų programos kodą.\n",
    "\n",
    "# [x] Done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0\n",
      "Loss =  5110.689639001278\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 52876 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  10\n",
      "Loss =  2311.6455734490687\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 75476 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  20\n",
      "Loss =  2060.4015232793877\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 77824 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  30\n",
      "Loss =  2036.0784662556812\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 75632 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  40\n",
      "Loss =  2033.4746907569347\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 74904 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  50\n",
      "Loss =  2033.1634891959918\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72972 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  60\n",
      "Loss =  2033.1224666056237\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72004 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  70\n",
      "Loss =  2033.1166547779553\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71932 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  80\n",
      "Loss =  2033.115792691137\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71840 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  90\n",
      "Loss =  2033.1156613671992\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70628 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  100\n",
      "Loss =  2033.115641068914\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70568 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  110\n",
      "Loss =  2033.115637907174\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70416 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  120\n",
      "Loss =  2033.1156374127052\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70668 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  130\n",
      "Loss =  2033.1156373352128\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70684 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  140\n",
      "Loss =  2033.1156373230558\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70780 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  150\n",
      "Loss =  2033.1156373211477\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70872 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  160\n",
      "Loss =  2033.1156373208487\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71116 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  170\n",
      "Loss =  2033.1156373208019\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70728 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  180\n",
      "Loss =  2033.1156373207946\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70548 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  190\n",
      "Loss =  2033.1156373207923\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70832 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  200\n",
      "Loss =  2033.1156373207916\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 71000 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  210\n",
      "Loss =  2033.1156373207912\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69696 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  220\n",
      "Loss =  2033.1156373207916\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69608 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  230\n",
      "Loss =  2033.1156373207912\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69496 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  240\n",
      "Loss =  2033.1156373207916\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69948 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  250\n",
      "Loss =  2033.1156373207912\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69828 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  260\n",
      "Loss =  2033.1156373207923\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69636 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  270\n",
      "Loss =  2033.1156373207923\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69672 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  280\n",
      "Loss =  2033.1156373207923\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69664 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  290\n",
      "Loss =  2033.1156373207928\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 70064 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2. Pakeiskite parametrus - mokymosi greiti, epochų skaičių, nepriklausomų kintamųjų skaičių,\n",
    "# pradinių duomenų pasiskirstymą.\n",
    "\n",
    "# Sample size\n",
    "M = 100\n",
    "\n",
    "# No. of input features\n",
    "n = 2\n",
    "\n",
    "# Learning Rate - Define during explanation\n",
    "l_r = 0.05\n",
    "\n",
    "# Number of iterations for updates - Define during explanation\n",
    "epochs = 300\n",
    "\n",
    "X, y = make_regression(n_samples=M, n_features=n, n_informative=n, n_targets=1, random_state=100, noise=1)\n",
    "y = np.reshape(y,(y.size, 1))\n",
    "\n",
    "train_and_show(X, y, epochs, l_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. Nustatykite, kokia nuostolių funkcija naudojama?\n",
    "\n",
    "# Ats.: MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Kodėl atnaujinant parametro  b reikšmę nedauginame iš X_train?\n",
    "\n",
    "# Ats.: Nes 'b' yra laisvas kintamasis ir jo reikšmė tiesiogiai nedauginama su 'xi' reikšme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0\n",
      "Loss =  31.87688937887143\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 55360 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  200\n",
      "Loss =  0.03017754359223175\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 81340 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  400\n",
      "Loss =  0.0029373331436930022\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 87712 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  600\n",
      "Loss =  0.001158735725441222\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 86916 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 5. Raskite realaus pasaulio duomenų rinkinį su tiesine priklausomybe ir pritaikykite šį modelį.\n",
    "\n",
    "def prepare_number_list(numbers):\n",
    "    return np.array(list(map(lambda value: [float(value)], numbers)))\n",
    "\n",
    "def normalize_column(column):\n",
    "    min_value = min(column)\n",
    "    max_value = max(column)\n",
    "\n",
    "    return map(lambda x: map_value(float(x), min_value, max_value, 0, 1), column)\n",
    "\n",
    "df = pd.read_csv('assets/Ice Cream Sales - temperatures.csv')\n",
    "X = prepare_number_list(normalize_column(df['Temperature'].values))\n",
    "y = prepare_number_list(normalize_column(df['Ice Cream Profits'].values))\n",
    "\n",
    "train_and_show(X, y, 800, 0.1, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0\n",
      "Loss =  404.76144508093915\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 47796 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  200\n",
      "Loss =  0.2645986409673728\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 93424 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  400\n",
      "Loss =  0.08472563651985762\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 125764 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  600\n",
      "Loss =  0.03461215467939087\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 171612 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  800\n",
      "Loss =  0.020650298258031264\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 166080 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  1000\n",
      "Loss =  0.016760458087850726\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 163120 characters]",
      "text/plain": [
       "<Figure size 1600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 6. Raskite arba sugeneruoti duomenų rinkinį su netiesine priklausomybe ir pritaikyti šį modelį.\n",
    "\n",
    "def prepare_number_list(numbers):\n",
    "    return np.array(list(map(lambda value: [float(value)], numbers)))\n",
    "\n",
    "def normalize_column(column):\n",
    "    min_value = min(column)\n",
    "    max_value = max(column)\n",
    "\n",
    "    return map(lambda x: map_value(float(x), min_value, max_value, 0, 1), column)\n",
    "\n",
    "df = pd.read_csv('assets/apple_quality.csv')\n",
    "X = prepare_number_list(normalize_column(df['Size'].values))\n",
    "y = prepare_number_list(normalize_column(df['Weight'].values))\n",
    "\n",
    "train_and_show(X, y, 1001, 0.1, 200)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main\n",
    "\n",
    "def relu(Z):\n",
    "    return np.maximum(0, Z)\n",
    "\n",
    "def sigmoid(Z):\n",
    "    return 1/(1+np.exp(-Z))\n",
    "\n",
    "def sigmoid_backward(dA, Z):\n",
    "    sig = sigmoid(Z)\n",
    "    return dA * sig * (1 - sig)\n",
    "\n",
    "def relu_backward(dA, Z):\n",
    "    dZ = np.array(dA, copy = True)\n",
    "    dZ[Z <= 0] = 0;\n",
    "    return dZ;\n",
    "\n",
    "@dataclass\n",
    "class NeuralNetworkLayer:\n",
    "    weights: np.array\n",
    "    bias: np.array\n",
    "    activation: Literal[\"relu\", \"sigmoid\"]\n",
    "\n",
    "    def get_activation_func(self):\n",
    "        if self.activation == \"relu\":\n",
    "            return relu\n",
    "        elif self.activation == \"sigmoid\":\n",
    "            return sigmoid\n",
    "        else:\n",
    "            raise Exception(f\"Non-supported activation function: '{self.activation}'\")\n",
    "\n",
    "    def get_backward_activation_func(self):\n",
    "        if self.activation == \"relu\":\n",
    "            return relu_backward\n",
    "        elif self.activation == \"sigmoid\":\n",
    "            return sigmoid_backward\n",
    "        else:\n",
    "            raise Exception(f\"Non-supported activation function: '{self.activation}'\")\n",
    "\n",
    "@dataclass\n",
    "class NeuralNetwork:\n",
    "    layers: list[NeuralNetworkLayer]\n",
    "\n",
    "def init_network(architecture, seed=1):\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    layers = []\n",
    "    for i in range(len(architecture) - 1):\n",
    "        layer_input_size = architecture[i][\"size\"]\n",
    "        layer_output_size = architecture[i+1][\"size\"]\n",
    "\n",
    "        layers.append(NeuralNetworkLayer(\n",
    "            np.random.randn(layer_output_size, layer_input_size) * 0.1,\n",
    "            np.random.randn(layer_output_size, 1) * 0.1,\n",
    "            architecture[i+1][\"activation\"]\n",
    "        ))\n",
    "\n",
    "    return NeuralNetwork(layers)\n",
    "\n",
    "def single_layer_forward_propagation(A_prev, W_curr, b_curr, activation_func):\n",
    "    Z_curr = np.dot(W_curr, A_prev) + b_curr\n",
    "\n",
    "    return activation_func(Z_curr), Z_curr\n",
    "\n",
    "def get_cost_value(Y_hat, Y):\n",
    "    # number of examples\n",
    "    m = Y_hat.shape[1]\n",
    "    # calculation of the cost according to the formula\n",
    "    cost = -1 / m * (np.dot(Y, np.log(Y_hat).T) + np.dot(1 - Y, np.log(1 - Y_hat).T))\n",
    "    return np.squeeze(cost)\n",
    "\n",
    "def convert_prob_into_class(probs):\n",
    "    probs_ = np.copy(probs)\n",
    "    probs_[probs_ > 0.5] = 1\n",
    "    probs_[probs_ <= 0.5] = 0\n",
    "    return probs_\n",
    "\n",
    "def get_accuracy_value(Y_hat, Y):\n",
    "    Y_hat_ = convert_prob_into_class(Y_hat)\n",
    "    return (Y_hat_ == Y).all(axis=0).mean()\n",
    "\n",
    "def full_forward_propagation(X, network: NeuralNetwork):\n",
    "    A_values = {}\n",
    "    Z_values = {}\n",
    "\n",
    "    A_curr = X\n",
    "    for idx, layer in enumerate(network.layers):\n",
    "        A_prev = A_curr\n",
    "\n",
    "        W_curr = layer.weights\n",
    "        b_curr = layer.bias\n",
    "        A_curr, Z_curr = single_layer_forward_propagation(A_prev, W_curr, b_curr, layer.get_activation_func())\n",
    "\n",
    "        A_values[idx - 1] = A_prev\n",
    "        Z_values[idx] = Z_curr\n",
    "\n",
    "    return A_curr, A_values, Z_values\n",
    "\n",
    "def single_layer_backward_propagation(dA_curr, W_curr, b_curr, Z_curr, A_prev, backward_activation_func):\n",
    "    # number of examples\n",
    "    m = A_prev.shape[1]\n",
    "\n",
    "    # calculation of the activation function derivative\n",
    "    dZ_curr = backward_activation_func(dA_curr, Z_curr)\n",
    "\n",
    "    # derivative of the matrix W\n",
    "    dW_curr = np.dot(dZ_curr, A_prev.T) / m\n",
    "    # derivative of the vector b\n",
    "    db_curr = np.sum(dZ_curr, axis=1, keepdims=True) / m\n",
    "    # derivative of the matrix A_prev\n",
    "    dA_prev = np.dot(W_curr.T, dZ_curr)\n",
    "\n",
    "    return dA_prev, dW_curr, db_curr\n",
    "\n",
    "def full_backward_propagation(Y_hat, Y, A_values, Z_values, network: NeuralNetwork):\n",
    "    dW = {}\n",
    "    db = {}\n",
    "\n",
    "    # number of examples\n",
    "    m = Y.shape[1]\n",
    "    # a hack ensuring the same shape of the prediction vector and labels vector\n",
    "    Y = Y.reshape(Y_hat.shape)\n",
    "\n",
    "    # initiation of gradient descent algorithm\n",
    "    dA_prev = -(np.divide(Y, Y_hat) - np.divide(1 - Y, 1 - Y_hat))\n",
    "\n",
    "    for layer_idx_curr, layer in reversed(list(enumerate(network.layers))):\n",
    "        layer_idx_prev = layer_idx_curr - 1\n",
    "\n",
    "        activation_func = layer.get_backward_activation_func()\n",
    "\n",
    "        dA_curr = dA_prev\n",
    "\n",
    "        A_prev = A_values[layer_idx_prev]\n",
    "        Z_curr = Z_values[layer_idx_curr]\n",
    "\n",
    "        W_curr = layer.weights\n",
    "        b_curr = layer.bias\n",
    "\n",
    "        dA_prev, dW_curr, db_curr = single_layer_backward_propagation(dA_curr, W_curr, b_curr, Z_curr, A_prev, activation_func)\n",
    "\n",
    "        dW[layer_idx_curr] = dW_curr\n",
    "        db[layer_idx_curr] = db_curr\n",
    "\n",
    "    return dW, db\n",
    "\n",
    "def update_network(network: NeuralNetwork, dW, db, learning_rate):\n",
    "    for layer_idx, layer in enumerate(network.layers, 0):\n",
    "        layer.weights -= learning_rate * dW[layer_idx]\n",
    "        layer.bias    -= learning_rate * db[layer_idx]\n",
    "\n",
    "def train(X_train, Y_train, network: NeuralNetwork, epochs, learning_rate, verbose=False, callback=None):\n",
    "    cost_history = []\n",
    "    accuracy_history = []\n",
    "\n",
    "    for i in range(epochs):\n",
    "        Y_hat, A_values, Z_values = full_forward_propagation(X_train, network)\n",
    "\n",
    "        # calculating metrics and saving them in history\n",
    "        cost = get_cost_value(Y_hat, Y_train)\n",
    "        cost_history.append(cost)\n",
    "        accuracy = get_accuracy_value(Y_hat, Y_train)\n",
    "        accuracy_history.append(accuracy)\n",
    "\n",
    "        # step backward - calculating gradient\n",
    "        dW, db = full_backward_propagation(Y_hat, Y_train, A_values, Z_values, network)\n",
    "        # updating model state\n",
    "        update_network(network, dW, db, learning_rate)\n",
    "\n",
    "        if(i % 50 == 0):\n",
    "            if(verbose):\n",
    "                print(\"Iteration: {:05} - cost: {:.5f} - accuracy: {:.5f}\".format(i, cost, accuracy))\n",
    "            if(callback is not None):\n",
    "                callback(i, network)\n",
    "\n",
    "    return cost_history, accuracy_history\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualization functions\n",
    "\n",
    "def make_plot(X, y, plot_name, file_name=None, XX=None, YY=None, preds=None, dark=False):\n",
    "    if (dark):\n",
    "        plt.style.use('dark_background')\n",
    "    else:\n",
    "        sns.set_style(\"whitegrid\")\n",
    "    plt.figure(figsize=(8,6))\n",
    "    axes = plt.gca()\n",
    "    axes.set(xlabel=\"$X_1$\", ylabel=\"$X_2$\")\n",
    "    plt.title(plot_name, fontsize=30)\n",
    "    plt.subplots_adjust(left=0.20)\n",
    "    plt.subplots_adjust(right=0.80)\n",
    "    if(XX is not None and YY is not None and preds is not None):\n",
    "        plt.contourf(XX, YY, preds.reshape(XX.shape), 25, alpha = 1, cmap=cm.Spectral)\n",
    "        plt.contour(XX, YY, preds.reshape(XX.shape), levels=[.5], cmap=\"Greys\", vmin=0, vmax=.6)\n",
    "\n",
    "    plt.scatter(X[:, 0], X[:, 1], c=y.ravel(), s=40, cmap=plt.cm.Spectral, edgecolors='black')\n",
    "    if(file_name):\n",
    "        plt.savefig(file_name)\n",
    "        plt.close()\n",
    "\n",
    "\n",
    "def viz_train_results(title, X, y, cost_history, accuracy_history, network: NeuralNetwork):\n",
    "    plt.plot(cost_history)\n",
    "    plt.title(\"Cost\")\n",
    "    plt.show()\n",
    "\n",
    "    plt.plot(accuracy_history)\n",
    "    plt.title(\"Accuracy\")\n",
    "    plt.show()\n",
    "\n",
    "    grid = np.mgrid[-1:2:100j,-1:2:100j]\n",
    "    grid_2d = grid.reshape(2, -1).T\n",
    "    XX, YY = grid\n",
    "\n",
    "    prediction_probs_numpy, _, _ = full_forward_propagation(grid_2d.T, network)\n",
    "    prediction_probs_numpy = prediction_probs_numpy.reshape(prediction_probs_numpy.shape[1], 1)\n",
    "    make_plot(X, y, title, file_name=None, XX=XX, YY=YY, preds=prediction_probs_numpy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Studijuokite (Lab23) pateiktą interaktyvų programos kodą.\n",
    "\n",
    "# [x] Done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23884 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18188 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34620 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2. Nubraižykite nuostolių funkcijos ir modelio tikslumo reikšmes.\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\": 50, \"activation\": \"relu\"},\n",
    "    {\"size\": 50, \"activation\": \"relu\"},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "X_train = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y_train = np.array([0,0,0,1])\n",
    "cost_history, accuracy_history = train(X_train.T, np.transpose(Y_train.reshape((Y_train.shape[0], 1))), network, 1000, 0.1)\n",
    "\n",
    "viz_train_results(\"AND\", X_train, Y_train, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24768 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 18248 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 35548 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 3. Pakeiskite tinklo architektūrą bei mokymosi greitį, epochų skaičių, analizuokite (2) punkte nurodytus grafikus.\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "X_train = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y_train = np.array([0,0,0,1])\n",
    "cost_history, accuracy_history = train(X_train.T, np.transpose(Y_train.reshape((Y_train.shape[0], 1))), network, 500, 0.2)\n",
    "\n",
    "viz_train_results(\"AND\", X_train, Y_train, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23376 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 17984 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 34396 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4.1. Pakeiskite tinklo apmokymo įmtį į kitas operacijas (OR, XOR) ir kartokite (3) punktą. (OR)\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "X_train = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y_train = np.array([0,1,1,1])\n",
    "cost_history, accuracy_history = train(X_train.T, np.transpose(Y_train.reshape((Y_train.shape[0], 1))), network, 500, 0.2)\n",
    "\n",
    "viz_train_results(\"OR\", X_train, Y_train, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0]\n",
      " [0 1]\n",
      " [1 0]\n",
      " [1 1]]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 26092 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16784 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50972 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 4.2. Pakeiskite tinklo apmokymo įmtį į kitas operacijas (OR, XOR) ir kartokite (3) punktą. (XOR)\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "X_train = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y_train = np.array([0,1,1,0])\n",
    "cost_history, accuracy_history = train(X_train.T, np.transpose(Y_train.reshape((Y_train.shape[0], 1))), network, 500, 0.2)\n",
    "\n",
    "print(X_train)\n",
    "\n",
    "viz_train_results(\"XOR\", X_train, Y_train, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 25016 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 19172 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 82556 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 5. Pakeiskite tinklo apmokymo įmtį į Moons duomenų rinkinį ir kartokite (3) punktą.\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "X, Y = make_moons(n_samples = 100, noise=0.2, random_state=100)\n",
    "\n",
    "cost_history, accuracy_history = train(X.T, np.transpose(Y.reshape((Y.shape[0], 1))), network, 1500, 0.20)\n",
    "\n",
    "viz_train_results(\"Moons\", X, Y, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ True  True False ... False  True  True]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 30092 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 28500 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ True  True False ... False  True  True]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 86792 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 6. Pamėginkite rasti realaus pasaulio duomenų rinkinį ir kartokite (3) punktą.\n",
    "\n",
    "def prepare_number_list(numbers):\n",
    "    return np.array(list(map(lambda value: [float(value)], numbers)))\n",
    "\n",
    "def normalize_column(column):\n",
    "    min_value = min(column)\n",
    "    max_value = max(column)\n",
    "\n",
    "    return map(lambda x: map_value(float(x), min_value, max_value, 0, 1), column)\n",
    "\n",
    "df = pd.read_csv('assets/apple_quality.csv')\n",
    "X0 = normalize_column(df['Juiciness'].values)\n",
    "X1 = normalize_column(df['Crunchiness'].values)\n",
    "X = np.array(list(zip(X0, X1)))\n",
    "Y = (df['Quality'] == 'good').values\n",
    "\n",
    "network = init_network(architecture = [\n",
    "    {\"size\":  2},\n",
    "    {\"size\": 25, \"activation\": \"relu\"},\n",
    "    {\"size\":  1, \"activation\": \"sigmoid\"}\n",
    "], seed=2)\n",
    "\n",
    "cost_history, accuracy_history = train(X.T, np.transpose(Y.reshape((Y.shape[0], 1))), network, 1000, 0.1)\n",
    "\n",
    "viz_train_results(\"Apple quality\", X, Y, cost_history, accuracy_history, network)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 6. Pakeiskite nuostolių funkciją į MSE, palyginkite rezultatus."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
