{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST digits dataset\n",
    "\n",
    "- First load and view the MNIST digits dataset\n",
    "- There are 60000 images in this dataset, but we will only view the first 25 of them:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From f:\\KTU\\Neuroninių tinklų metodai\\venv\\Lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n",
      "test_images shape (10000, 28, 28) train_images shape (60000, 28, 28)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 36964 characters]",
      "text/plain": [
       "<Figure size 1000x1000 with 25 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load and visualise the MNIST digits\n",
    "import tensorflow as tf\n",
    "tf.config.experimental.set_visible_devices([], \"GPU\")\n",
    "\n",
    "mnist = tf.keras.datasets.mnist\n",
    "(train_images0, train_labels0),(test_images0, test_labels0) = mnist.load_data()\n",
    "\n",
    "print(\"test_images shape\",test_images0.shape,\"train_images shape\",train_images0.shape)\n",
    "class_names=[\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\"]\n",
    "import matplotlib.pyplot as plt\n",
    "# plot first few images\n",
    "plt.figure(figsize=(10,10))\n",
    "for i in range(25):\n",
    "    # define subplot\n",
    "    plt.subplot(5,5,i+1)\n",
    "    # plot raw pixel data\n",
    "    plt.imshow(train_images0[i], cmap=plt.get_cmap('gray'))\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.grid(False)\n",
    "    # Add a label underneath...\n",
    "    plt.xlabel(class_names[train_labels0[i]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Next build a neural-network classifier for these digits.\n",
    "\n",
    "- We will build a keras model, with the higher-level API concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "# Each MNIST images are 28*28.  Therefore if there are N images, then the\n",
    "# shape of the numpy array holding the images is N*28*28\n",
    "# We will reshape that here to be N*784, using a numpy reshape.\n",
    "# Note that this flattens each image into a single vector length 784.\n",
    "#test_images=test_images0.reshape(10000,784) # 10000 test patterns\n",
    "#train_images=train_images0.reshape(60000,784) # 60000 train patterns\n",
    "\n",
    "test_images=test_images0\n",
    "train_images=train_images0\n",
    "\n",
    "# Also rescale greyscale from 8 bit to floating point (by dividing by 255)\n",
    "test_images=test_images/255.0\n",
    "train_images=train_images/255.0\n",
    "\n",
    "# Create the model\n",
    "\n",
    "#model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))\n",
    "#model.add(layers.MaxPooling2D((2, 2)))\n",
    "#model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "#model.add(layers.MaxPooling2D((2, 2)))\n",
    "#model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "\n",
    "\n",
    "keras_model = tf.keras.models.Sequential([\n",
    "tf.keras.layers.Conv2D(28, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
    "tf.keras.layers.Dropout(.2, input_shape=(2,)),\n",
    "tf.keras.layers.MaxPooling2D((2, 2)),\n",
    "tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n",
    "tf.keras.layers.Dropout(.2, input_shape=(2,)),\n",
    "tf.keras.layers.MaxPooling2D((2, 2)),\n",
    "tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n",
    "tf.keras.layers.Dropout(.2, input_shape=(2,)),\n",
    "tf.keras.layers.Flatten(),\n",
    "tf.keras.layers.Dense(64, activation='relu'),\n",
    "tf.keras.layers.Dense(10, activation='softmax')\n",
    "])\n",
    "\n",
    "keras_model.build(input_shape=[None,784])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## View the keras model summary information\n",
    "\n",
    "- This shows you how many layers your neural network has, and how many weights, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv2d_6 (Conv2D)           (None, 26, 26, 28)        280       \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 26, 26, 28)        0         \n",
      "                                                                 \n",
      " max_pooling2d_4 (MaxPooling  (None, 13, 13, 28)       0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_7 (Conv2D)           (None, 11, 11, 64)        16192     \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 11, 11, 64)        0         \n",
      "                                                                 \n",
      " max_pooling2d_5 (MaxPooling  (None, 5, 5, 64)         0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_8 (Conv2D)           (None, 3, 3, 64)          36928     \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 3, 3, 64)          0         \n",
      "                                                                 \n",
      " flatten_2 (Flatten)         (None, 576)               0         \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 64)                36928     \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 10)                650       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 90,978\n",
      "Trainable params: 90,978\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# View the model summary information...\n",
    "keras_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Keras model\n",
    "\n",
    "- We will use SGD optimiser (ordinary gradient descent)\n",
    "- We will use Cross Entropy loss (\"SparseCategoricalCrossentropy\")\n",
    "- We will run 200 training iterations (epochs)..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "WARNING:tensorflow:From f:\\KTU\\Neuroninių tinklų metodai\\venv\\Lib\\site-packages\\keras\\src\\utils\\tf_utils.py:492: The name tf.ragged.RaggedTensorValue is deprecated. Please use tf.compat.v1.ragged.RaggedTensorValue instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\KTU\\Neuroninių tinklų metodai\\venv\\Lib\\site-packages\\keras\\src\\backend.py:5727: UserWarning: \"`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a Softmax activation and thus does not represent logits. Was this intended?\n",
      "  output, from_logits = _get_logits(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "600/600 [==============================] - 14s 21ms/step - loss: 0.2375 - sparse_categorical_accuracy: 0.9275 - val_loss: 0.0672 - val_sparse_categorical_accuracy: 0.9833\n",
      "Epoch 2/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0696 - sparse_categorical_accuracy: 0.9789 - val_loss: 0.0431 - val_sparse_categorical_accuracy: 0.9901\n",
      "Epoch 3/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0521 - sparse_categorical_accuracy: 0.9839 - val_loss: 0.0403 - val_sparse_categorical_accuracy: 0.9903\n",
      "Epoch 4/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0408 - sparse_categorical_accuracy: 0.9874 - val_loss: 0.0274 - val_sparse_categorical_accuracy: 0.9927\n",
      "Epoch 5/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0356 - sparse_categorical_accuracy: 0.9887 - val_loss: 0.0294 - val_sparse_categorical_accuracy: 0.9921\n",
      "Epoch 6/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0309 - sparse_categorical_accuracy: 0.9901 - val_loss: 0.0280 - val_sparse_categorical_accuracy: 0.9911\n",
      "Epoch 7/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0288 - sparse_categorical_accuracy: 0.9909 - val_loss: 0.0222 - val_sparse_categorical_accuracy: 0.9942\n",
      "Epoch 8/20\n",
      "600/600 [==============================] - 14s 23ms/step - loss: 0.0261 - sparse_categorical_accuracy: 0.9913 - val_loss: 0.0225 - val_sparse_categorical_accuracy: 0.9937\n",
      "Epoch 9/20\n",
      "600/600 [==============================] - 15s 24ms/step - loss: 0.0220 - sparse_categorical_accuracy: 0.9929 - val_loss: 0.0260 - val_sparse_categorical_accuracy: 0.9927\n",
      "Epoch 10/20\n",
      "600/600 [==============================] - 15s 24ms/step - loss: 0.0207 - sparse_categorical_accuracy: 0.9934 - val_loss: 0.0217 - val_sparse_categorical_accuracy: 0.9940\n",
      "Epoch 11/20\n",
      "600/600 [==============================] - 14s 24ms/step - loss: 0.0198 - sparse_categorical_accuracy: 0.9934 - val_loss: 0.0225 - val_sparse_categorical_accuracy: 0.9926\n",
      "Epoch 12/20\n",
      "600/600 [==============================] - 15s 25ms/step - loss: 0.0169 - sparse_categorical_accuracy: 0.9945 - val_loss: 0.0214 - val_sparse_categorical_accuracy: 0.9930\n",
      "Epoch 13/20\n",
      "600/600 [==============================] - 14s 23ms/step - loss: 0.0169 - sparse_categorical_accuracy: 0.9943 - val_loss: 0.0235 - val_sparse_categorical_accuracy: 0.9929\n",
      "Epoch 14/20\n",
      "600/600 [==============================] - 14s 24ms/step - loss: 0.0149 - sparse_categorical_accuracy: 0.9949 - val_loss: 0.0235 - val_sparse_categorical_accuracy: 0.9933\n",
      "Epoch 15/20\n",
      "600/600 [==============================] - 15s 25ms/step - loss: 0.0145 - sparse_categorical_accuracy: 0.9954 - val_loss: 0.0205 - val_sparse_categorical_accuracy: 0.9939\n",
      "Epoch 16/20\n",
      "600/600 [==============================] - 15s 24ms/step - loss: 0.0143 - sparse_categorical_accuracy: 0.9954 - val_loss: 0.0217 - val_sparse_categorical_accuracy: 0.9934\n",
      "Epoch 17/20\n",
      "600/600 [==============================] - 14s 23ms/step - loss: 0.0128 - sparse_categorical_accuracy: 0.9956 - val_loss: 0.0222 - val_sparse_categorical_accuracy: 0.9938\n",
      "Epoch 18/20\n",
      "600/600 [==============================] - 13s 21ms/step - loss: 0.0132 - sparse_categorical_accuracy: 0.9955 - val_loss: 0.0225 - val_sparse_categorical_accuracy: 0.9939\n",
      "Epoch 19/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0106 - sparse_categorical_accuracy: 0.9966 - val_loss: 0.0224 - val_sparse_categorical_accuracy: 0.9933\n",
      "Epoch 20/20\n",
      "600/600 [==============================] - 13s 22ms/step - loss: 0.0118 - sparse_categorical_accuracy: 0.9961 - val_loss: 0.0214 - val_sparse_categorical_accuracy: 0.9939\n"
     ]
    }
   ],
   "source": [
    "keras_model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(0.001),\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n",
    ")\n",
    "\n",
    "# Train loop\n",
    "history = keras_model.fit(\n",
    "    train_images,\n",
    "    train_labels0,\n",
    "    batch_size=100,\n",
    "    epochs=20,\n",
    "    validation_data=(test_images, test_labels0),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## View the training performance\n",
    "\n",
    "- When the Keras fit loop runs, it returns a \"history\" object, which includes a dictionary of the trianing history.\n",
    "\n",
    "- Hence we can plot graphs of the training performance (Accuracy, Loss), for both the \"Training\" and \"Validation\" sets...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key: loss\n",
      "Key: sparse_categorical_accuracy\n",
      "Key: val_loss\n",
      "Key: val_sparse_categorical_accuracy\n"
     ]
    }
   ],
   "source": [
    "# first show keys for data series recorded by fit loop:\n",
    "for item in history.history:\n",
    "    print(\"Key:\",item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 38228 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 38536 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(history.history['loss'],label=\"train\")\n",
    "plt.plot(history.history['val_loss'],label=\"validation\")\n",
    "plt.title('Model Loss')\n",
    "plt.yscale('log')\n",
    "plt.ylabel('Cross Entropy')\n",
    "plt.xlabel('Iteration')\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "#print(\"history\",history.history)\n",
    "plt.plot(history.history['sparse_categorical_accuracy'],label=\"train\")\n",
    "plt.plot(history.history['val_sparse_categorical_accuracy'],label=\"validation\")\n",
    "plt.title('Model Accuracy')\n",
    "#plt.yscale('log')\n",
    "plt.ylabel('Acc')\n",
    "plt.xlabel('Iteration')\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspect how well the system is working on a sample of 25 new images (from the test set)...\n",
    "- The test set has a lot of images in it, but we can only view 25 at a time.\n",
    "- Hence rerun this code block several times, to get a different random set of samples from the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54848 characters]",
      "text/plain": [
       "<Figure size 1000x1000 with 25 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "plt.figure(figsize=(10,10))\n",
    "# plot 25 random images from the test set.\n",
    "first_index=np.random.randint(len(test_images)-25)\n",
    "for i in range(first_index,first_index+25):\n",
    "    # define subplot\n",
    "    plt.subplot(5,5,i+1-first_index)\n",
    "    # plot raw pixel data\n",
    "    plt.imshow(test_images0[i], cmap=plt.get_cmap('gray'))\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "    plt.grid(False)\n",
    "    if class_names!=None:\n",
    "        prediction=keras_model(test_images[i:i+1])[0,:] # This will be a vector of length 10\n",
    "        prediction_class=np.argmax(prediction)  # Pick the index of the largest element of the length-10 vector\n",
    "        # Add a label underneath...\n",
    "        true_label=test_labels0[i]\n",
    "        class_name=class_names[prediction_class]\n",
    "        plt.xlabel(class_name+\" \"+(\"CORRECT\" if prediction_class==true_label else \"WRONG\"))\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
