Machine Learning

Convolutional Neural Network

2020, May 05     GGabry

Image classification with tensorflow and Convolutional Neural Netorks

Here we load the data and the rest of the code

import os
import sys
import tensorflow as tf
from google.colab import drive

path = "/content/drive"
drive.mount(path, force_remount=True)

mypath = "drive/MyDrive/colab/img_classification"
mydata = os.path.join(mypath, "data")
import glob
myfiles = glob.glob(os.path.join(mypath, '*'))
print(myfiles)
print(mydata)

Mounted at /content/drive
['drive/MyDrive/colab/img_classification/model.py', 'drive/MyDrive/colab/img_classification/solver.py', 'drive/MyDrive/colab/img_classification/data', 'drive/MyDrive/colab/img_classification/__pycache__', 'drive/MyDrive/colab/img_classification/main.py', 'drive/MyDrive/colab/img_classification/data_loader.py', 'drive/MyDrive/colab/img_classification/Image_classification.ipynb']
drive/MyDrive/colab/img_classification/data

Check if gpu is connected

## data_loader.py here for rendering purpose
import pickle
import os
import glob
import cv2
import numpy as np
import random
import tensorflow as tf

def load_stl(data_path = '/data/datasets/stl10', norm=True):
    def normalize(data):
        data = (data / 255) - 0.5
        return data

    def load_data(mypath):
        data = np.zeros((0, 96, 96, 3), dtype='uint8')
        labels = np.zeros((0,))
        for i, cla in enumerate(mypath):
            filelist = glob.glob(os.path.join(cla, '*.png'))
            tmp_data = np.empty((len(filelist), 96, 96, 3), dtype='uint8')
            tmp_labels = np.ones((len(filelist),)) * i
            for j, path in enumerate(filelist):
                image = cv2.imread(path)
                tmp_data[j, :] = image
            data = np.concatenate((data, tmp_data))
            labels = np.concatenate((labels, tmp_labels))
        return data, labels

    train_path = glob.glob(os.path.join(data_path, 'train', '*'))
    train_path.sort()
    print(train_path)
    test_path = glob.glob(os.path.join(data_path, 'test', '*'))
    test_path.sort()
    training_data, training_labels = load_data(train_path)
    test_data, test_labels = load_data(test_path)
    perm = np.arange(test_data.shape[0])
    random.shuffle(perm)
    perm = perm[:1000]
    test_data = test_data[perm, :]
    test_labels = test_labels[perm]
    if norm:
        training_data = normalize(training_data)
        test_data = normalize(test_data)
    return training_data, training_labels, test_data, test_labels


def load_cifar10():
    def unpickle_cifar10(path):
        files = glob.glob(path)
        od = np.zeros((0, 32, 32, 3), dtype='uint8')
        ol = np.zeros((0,))
        for i in files:
            with open(i, 'rb') as fo:
                mydict = pickle.load(fo, encoding='bytes')
            data = mydict[b'data']
            data = data.reshape(data.shape[0], 3, 32, 32).transpose(0, 2, 3, 1).astype("uint8")
            labels = np.asarray(mydict[b'labels'])
            od = np.concatenate((od, data))
            ol = np.concatenate((ol, labels))
        return od, ol
    data_path = '/data/datasets/cifar10/cifar-10-batches-py'
    training_data, training_labels = unpickle_cifar10(os.path.join(data_path, 'data*'))
    test_data, test_labels = unpickle_cifar10(os.path.join(data_path, 'test*'))
    return training_data, training_labels, test_data, test_labels


def load_cifar100():
    def unpickle(file):
        with open(file, 'rb') as fo:
            mydict = pickle.load(fo, encoding='bytes')
        data = mydict[b'data']
        data = data.reshape(data.shape[0], 3, 32, 32).transpose(0, 2, 3, 1).astype("uint8")
        labels = np.asarray(mydict[b'coarse_labels'])
        return data, labels
    data_path = '/data/datasets/cifar-100-python'
    training_data, training_labels = unpickle(os.path.join(data_path, 'train'))
    test_data, test_labels = unpickle(os.path.join(data_path, 'test'))
    return training_data, training_labels, test_data, test_labels


def load_visual_small(data_path='/data/datasets/visual_small'):
    def load_data(mypath):
        data = np.zeros((0, 150, 150, 3), dtype='uint8')
        labels = np.zeros((0,))
        for i, cla in enumerate(mypath):
            filelist = glob.glob(os.path.join(cla, '*.jpg'))
            tmp_data = np.empty((len(filelist), 150, 150, 3), dtype='uint8')
            tmp_labels = np.ones((len(filelist),)) * i
            for j, path in enumerate(filelist):
                image = cv2.imread(path)
                image = cv2.resize(image, (150, 150))
                tmp_data[j, :] = image
            data = np.concatenate((data, tmp_data))
            labels = np.concatenate((labels, tmp_labels))
        return data, labels
    train_path = glob.glob(os.path.join(data_path, 'train', '*'))
    train_path.sort()
    test_path = glob.glob(os.path.join(data_path, 'test', '*'))
    test_path.sort()
    training_data, training_labels = load_data(train_path)
    test_data, test_labels = load_data(test_path)
    return training_data, training_labels, test_data, test_labels

import tensorflow as tf
import sys
from data_loader import load_stl
from model import Densenet, MobileNet, ResNet
from solver import ImageClassifier



def main():
    if tf.test.gpu_device_name():
        print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
    else:
        print("Please install GPU version of TF")
        sys.exit(1)
    training_data, training_labels, test_data, test_labels = load_stl()
    model = ResNet()
    ic = ImageClassifier(batch_size=32, epochs=100)
    ic.train(training_data, training_labels, test_data, test_labels, model)


# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    main()

import tensorflow as tf
from tensorflow.keras import Model
from tensorflow.keras.layers import Layer


class Densenet(Model):
    def __init__(self):
        super(Densenet, self).__init__(trainable=True)
        self.backbone = tf.keras.applications.DenseNet121(include_top=False, input_shape=[96, 96, 3])
        self.final_conv = tf.keras.layers.Conv2D(filters=10, kernel_size=[3, 3], padding='valid')
        self.final_flatten = tf.keras.layers.Flatten()

    def call(self, inputs, training=None, mask=None):
        x = self.backbone(inputs)
        x = self.final_conv(x)
        x = self.final_flatten(x)
        return x, tf.keras.activations.softmax(x)


class MobileNet(Model):
    def __init__(self):
        super(MobileNet, self).__init__(trainable=True)
        self.backbone = tf.keras.applications.MobileNetV2(include_top=False, input_shape=[96, 96, 3])
        self.final_conv = tf.keras.layers.Conv2D(filters=10, kernel_size=[3, 3], padding='valid')
        self.final_flatten = tf.keras.layers.Flatten()

    def call(self, inputs, training=None, mask=None):
        x = self.backbone(inputs)
        x = self.final_conv(x)
        x = self.final_flatten(x)
        return x, tf.keras.activations.softmax(x)

class ResNet(Model):
    def __init__(self):
        super(ResNet, self).__init__(trainable=True)
        self.backbone = tf.keras.applications.ResNet50(include_top=False, input_shape=[96, 96, 3])
        self.final_conv = tf.keras.layers.Conv2D(filters=10, kernel_size=[3, 3], padding='valid')
        self.final_flatten = tf.keras.layers.Flatten()

    def call(self, inputs, training=None, mask=None):
        x = self.backbone(inputs)
        x = self.final_conv(x)
        x = self.final_flatten(x)
        return x, tf.keras.activations.softmax(x)

import tensorflow as tf
import random
import numpy as np


class ImageClassifier():
    def __init__(self, batch_size, epochs, ilr=0.0001):
        self.epochs = epochs
        self.batch_size = batch_size
        self.initial_learning_rate = ilr

    def train(self, training_data, training_labels, test_data, test_labels, model):
        supervised_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
        optimizer = tf.keras.optimizers.Adam(learning_rate=self.initial_learning_rate)

        @tf.function
        def train_step(data, labels):
            with tf.GradientTape() as tape:
                logits, preds = model(data, training=True)
                loss = supervised_loss(y_true=labels, y_pred=logits)
            trainable_vars = model.trainable_variables
            gradients = tape.gradient(loss, trainable_vars)
            optimizer.apply_gradients(zip(gradients, trainable_vars))
            eq = tf.equal(labels, tf.argmax(preds, -1))
            accuracy = tf.reduce_mean(tf.cast(eq, tf.float32)) * 100
            return loss, accuracy

        @tf.function
        def test_step(data, labels):
            logits, preds = model(data, training=False)
            loss = supervised_loss(y_true=labels, y_pred=logits)
            return loss, logits, preds

        global_step = 0
        best_accuracy = 0.0
        for e in range(self.epochs):

            ## Shuffling training set
            perm = np.arange(len(training_labels))
            random.shuffle(perm)
            training_data = training_data[perm]
            training_labels = training_labels[perm]

            ## Iteration
            for i in range(0, len(training_labels), self.batch_size):
                data = training_data[i:i+self.batch_size, :]
                labels = training_labels[i:i+self.batch_size, ].astype('int64')
                global_step += 1  # len(labels)
                batch_loss, batch_accuracy = train_step(data, labels)
                if global_step % 50 == 0:
                    print('[{0}-{1:03}] loss: {2:0.05}, batch_accuracy: {3:0.03}'.format(
                        e + 1, global_step,
                        batch_loss.numpy(),
                        batch_accuracy.numpy()))
                if global_step == 1:
                    print('number of model parameters {}'.format(model.count_params()))

            # Test the whole test dataset
            test_preds = tf.zeros((0,), dtype=tf.int64)
            total_loss = list()
            for i in range(0, len(test_labels), self.batch_size):
                data = test_data[i:i+self.batch_size, :]
                labels = test_labels[i:i+self.batch_size, ].astype('int64')
                batch_loss, _, preds = test_step(data, labels)
                batch_preds = tf.argmax(preds, -1)
                test_preds = tf.concat([test_preds, batch_preds], axis=0)
                total_loss.append(batch_loss)
            loss = sum(total_loss)/len(total_loss)
            eq = tf.equal(test_labels, test_preds)
            test_accuracy = tf.reduce_mean(tf.cast(eq, tf.float32)) * 100
            if test_accuracy > best_accuracy:
                best_accuracy = test_accuracy
            print('End of Epoch {0}/{1:03} -> loss: {2:0.05}, test accuracy: {3:0.03} - best accuracy: {4:0.03}'.format(
                        e + 1, self.epochs,
                        loss.numpy(),
                        test_accuracy.numpy(),
                        best_accuracy))

import sys
sys.path.append(mypath)
# from solver import ImageClassifier
# from model import ResNet
import pickle

if tf.test.gpu_device_name():
  print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
else:
  print("Please install GPU version of TF")
  sys.exit(1)

Default GPU Device: /device:GPU:0

Load data from the drive mydata

with open(os.path.join(mydata, 'stl.pkl'), 'rb') as f:
  training_data = pickle.load(f, encoding='bytes')
  training_labels = pickle.load(f, encoding='bytes')
  test_data = pickle.load(f, encoding='bytes')
  test_labels = pickle.load(f, encoding='bytes')

print(f'Training data shape: {training_data.shape}, test data shape: {test_data.shape}')

Training data shape: (5000, 96, 96, 3), test data shape: (1000, 96, 96, 3)

Instantiate the network, the solver and perform training

model = ResNet()
ic = ImageClassifier(batch_size=32, epochs=100)
ic.train(training_data, training_labels, test_data, test_labels, model)

Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5
94773248/94765736 [==============================] - 1s 0us/step
number of model parameters 23772042
[1-050] loss: 1.1618, batch_accuracy: 68.8
[1-100] loss: 0.56554, batch_accuracy: 84.4
[1-150] loss: 0.62058, batch_accuracy: 78.1
End of Epoch 1/100 -> loss: 3.0582, test accuracy: 10.7 - best accuracy: 10.7
[2-200] loss: 0.13438, batch_accuracy: 93.8
[2-250] loss: 0.38512, batch_accuracy: 87.5
[2-300] loss: 0.036739, batch_accuracy: 1e+02
End of Epoch 2/100 -> loss: 2.4779, test accuracy: 11.2 - best accuracy: 11.2
[3-350] loss: 0.0037163, batch_accuracy: 1e+02
[3-400] loss: 0.005563, batch_accuracy: 1e+02
[3-450] loss: 0.030149, batch_accuracy: 1e+02
End of Epoch 3/100 -> loss: 2.7729, test accuracy: 14.4 - best accuracy: 14.4
[4-500] loss: 0.0021009, batch_accuracy: 1e+02
[4-550] loss: 0.013833, batch_accuracy: 1e+02
[4-600] loss: 0.017271, batch_accuracy: 1e+02
End of Epoch 4/100 -> loss: 2.8077, test accuracy: 25.9 - best accuracy: 25.9
[5-650] loss: 0.0025531, batch_accuracy: 1e+02
[5-700] loss: 0.0037249, batch_accuracy: 1e+02
[5-750] loss: 0.0083606, batch_accuracy: 1e+02
End of Epoch 5/100 -> loss: 2.1405, test accuracy: 47.5 - best accuracy: 47.5
[6-800] loss: 0.0067636, batch_accuracy: 1e+02
[6-850] loss: 0.0072196, batch_accuracy: 1e+02
[6-900] loss: 0.004423, batch_accuracy: 1e+02
End of Epoch 6/100 -> loss: 1.1154, test accuracy: 70.6 - best accuracy: 70.6
[7-950] loss: 0.011483, batch_accuracy: 1e+02
[7-1000] loss: 0.0049225, batch_accuracy: 1e+02
[7-1050] loss: 0.014598, batch_accuracy: 1e+02
End of Epoch 7/100 -> loss: 0.6467, test accuracy: 82.9 - best accuracy: 82.9
[8-1100] loss: 0.001611, batch_accuracy: 1e+02
[8-1150] loss: 7.9496e-05, batch_accuracy: 1e+02
[8-1200] loss: 0.00051048, batch_accuracy: 1e+02
[8-1250] loss: 0.0049545, batch_accuracy: 1e+02
End of Epoch 8/100 -> loss: 0.53891, test accuracy: 86.5 - best accuracy: 86.5
[9-1300] loss: 0.0011814, batch_accuracy: 1e+02
[9-1350] loss: 0.049983, batch_accuracy: 96.9
[9-1400] loss: 0.027939, batch_accuracy: 1e+02
End of Epoch 9/100 -> loss: 0.56412, test accuracy: 85.3 - best accuracy: 86.5
[10-1450] loss: 0.0022441, batch_accuracy: 1e+02
[10-1500] loss: 0.036254, batch_accuracy: 96.9
[10-1550] loss: 0.006932, batch_accuracy: 1e+02
End of Epoch 10/100 -> loss: 0.86483, test accuracy: 82.1 - best accuracy: 86.5
[11-1600] loss: 0.087986, batch_accuracy: 96.9
[11-1650] loss: 0.18709, batch_accuracy: 90.6
[11-1700] loss: 0.26655, batch_accuracy: 93.8
End of Epoch 11/100 -> loss: 1.0175, test accuracy: 80.6 - best accuracy: 86.5
[12-1750] loss: 0.14483, batch_accuracy: 93.8
[12-1800] loss: 0.0076313, batch_accuracy: 1e+02
[12-1850] loss: 0.22575, batch_accuracy: 93.8
End of Epoch 12/100 -> loss: 0.6508, test accuracy: 83.9 - best accuracy: 86.5
[13-1900] loss: 0.10353, batch_accuracy: 96.9
[13-1950] loss: 0.099498, batch_accuracy: 96.9
[13-2000] loss: 0.13089, batch_accuracy: 96.9
End of Epoch 13/100 -> loss: 0.66508, test accuracy: 83.2 - best accuracy: 86.5
[14-2050] loss: 0.030181, batch_accuracy: 1e+02
[14-2100] loss: 0.0061268, batch_accuracy: 1e+02
[14-2150] loss: 0.0070929, batch_accuracy: 1e+02
End of Epoch 14/100 -> loss: 0.60802, test accuracy: 84.8 - best accuracy: 86.5
[15-2200] loss: 0.0035261, batch_accuracy: 1e+02
[15-2250] loss: 0.0046011, batch_accuracy: 1e+02
[15-2300] loss: 0.099044, batch_accuracy: 93.8
[15-2350] loss: 0.0098661, batch_accuracy: 1e+02
End of Epoch 15/100 -> loss: 0.53285, test accuracy: 85.8 - best accuracy: 86.5
[16-2400] loss: 0.0022657, batch_accuracy: 1e+02
[16-2450] loss: 0.00071664, batch_accuracy: 1e+02
[16-2500] loss: 0.0012597, batch_accuracy: 1e+02
End of Epoch 16/100 -> loss: 0.44718, test accuracy: 87.4 - best accuracy: 87.4
[17-2550] loss: 0.0031438, batch_accuracy: 1e+02
[17-2600] loss: 0.0018411, batch_accuracy: 1e+02
[17-2650] loss: 0.0031068, batch_accuracy: 1e+02
End of Epoch 17/100 -> loss: 0.47998, test accuracy: 88.0 - best accuracy: 88.0
[18-2700] loss: 0.00021323, batch_accuracy: 1e+02
[18-2750] loss: 7.3521e-05, batch_accuracy: 1e+02
[18-2800] loss: 0.00017758, batch_accuracy: 1e+02
End of Epoch 18/100 -> loss: 0.46432, test accuracy: 88.3 - best accuracy: 88.3
[19-2850] loss: 0.0003346, batch_accuracy: 1e+02
[19-2900] loss: 0.0013211, batch_accuracy: 1e+02
[19-2950] loss: 0.0013585, batch_accuracy: 1e+02
End of Epoch 19/100 -> loss: 0.50178, test accuracy: 87.8 - best accuracy: 88.3
[20-3000] loss: 0.00067244, batch_accuracy: 1e+02
[20-3050] loss: 0.00041101, batch_accuracy: 1e+02
[20-3100] loss: 0.00014497, batch_accuracy: 1e+02
End of Epoch 20/100 -> loss: 0.4583, test accuracy: 87.9 - best accuracy: 88.3
[21-3150] loss: 0.0048208, batch_accuracy: 1e+02
[21-3200] loss: 0.043673, batch_accuracy: 96.9
[21-3250] loss: 0.00020083, batch_accuracy: 1e+02
End of Epoch 21/100 -> loss: 0.68186, test accuracy: 85.2 - best accuracy: 88.3
[22-3300] loss: 0.00082463, batch_accuracy: 1e+02
[22-3350] loss: 0.00021113, batch_accuracy: 1e+02
[22-3400] loss: 0.030058, batch_accuracy: 96.9
[22-3450] loss: 0.098482, batch_accuracy: 96.9
End of Epoch 22/100 -> loss: 0.62557, test accuracy: 84.0 - best accuracy: 88.3
[23-3500] loss: 0.015707, batch_accuracy: 1e+02
[23-3550] loss: 0.00041413, batch_accuracy: 1e+02
[23-3600] loss: 0.0011022, batch_accuracy: 1e+02
End of Epoch 23/100 -> loss: 0.60242, test accuracy: 84.9 - best accuracy: 88.3
[24-3650] loss: 0.00052621, batch_accuracy: 1e+02
[24-3700] loss: 0.00054835, batch_accuracy: 1e+02
[24-3750] loss: 0.0010864, batch_accuracy: 1e+02
End of Epoch 24/100 -> loss: 0.56965, test accuracy: 86.5 - best accuracy: 88.3
[25-3800] loss: 0.026033, batch_accuracy: 1e+02
[25-3850] loss: 0.00035403, batch_accuracy: 1e+02
[25-3900] loss: 0.0051908, batch_accuracy: 1e+02
End of Epoch 25/100 -> loss: 0.84857, test accuracy: 82.5 - best accuracy: 88.3
[26-3950] loss: 0.0013316, batch_accuracy: 1e+02
[26-4000] loss: 0.045541, batch_accuracy: 96.9
[26-4050] loss: 0.0075176, batch_accuracy: 1e+02
End of Epoch 26/100 -> loss: 0.64193, test accuracy: 84.6 - best accuracy: 88.3
[27-4100] loss: 0.001776, batch_accuracy: 1e+02
[27-4150] loss: 0.050592, batch_accuracy: 96.9
[27-4200] loss: 0.00079038, batch_accuracy: 1e+02
End of Epoch 27/100 -> loss: 0.81409, test accuracy: 80.9 - best accuracy: 88.3
[28-4250] loss: 0.025884, batch_accuracy: 1e+02
[28-4300] loss: 0.055791, batch_accuracy: 96.9
[28-4350] loss: 0.27721, batch_accuracy: 87.5
End of Epoch 28/100 -> loss: 13.119, test accuracy: 49.9 - best accuracy: 88.3
[29-4400] loss: 0.46001, batch_accuracy: 90.6
[29-4450] loss: 0.24071, batch_accuracy: 87.5
[29-4500] loss: 0.28114, batch_accuracy: 90.6
[29-4550] loss: 0.18585, batch_accuracy: 90.6
End of Epoch 29/100 -> loss: 0.70831, test accuracy: 81.5 - best accuracy: 88.3
[30-4600] loss: 0.0036856, batch_accuracy: 1e+02
[30-4650] loss: 0.0087853, batch_accuracy: 1e+02
[30-4700] loss: 0.0071221, batch_accuracy: 1e+02
End of Epoch 30/100 -> loss: 0.55171, test accuracy: 84.3 - best accuracy: 88.3
[31-4750] loss: 0.00045587, batch_accuracy: 1e+02
[31-4800] loss: 0.0022662, batch_accuracy: 1e+02
[31-4850] loss: 0.020216, batch_accuracy: 1e+02
End of Epoch 31/100 -> loss: 0.72389, test accuracy: 84.4 - best accuracy: 88.3
[32-4900] loss: 0.00099202, batch_accuracy: 1e+02
[32-4950] loss: 0.0018475, batch_accuracy: 1e+02
[32-5000] loss: 0.04461, batch_accuracy: 96.9
End of Epoch 32/100 -> loss: 0.62957, test accuracy: 84.2 - best accuracy: 88.3
[33-5050] loss: 0.001073, batch_accuracy: 1e+02
[33-5100] loss: 0.0062568, batch_accuracy: 1e+02
[33-5150] loss: 0.00024773, batch_accuracy: 1e+02
End of Epoch 33/100 -> loss: 0.57837, test accuracy: 85.5 - best accuracy: 88.3
[34-5200] loss: 0.00087715, batch_accuracy: 1e+02
[34-5250] loss: 0.00024628, batch_accuracy: 1e+02
[34-5300] loss: 0.00027674, batch_accuracy: 1e+02
End of Epoch 34/100 -> loss: 0.59281, test accuracy: 86.3 - best accuracy: 88.3
[35-5350] loss: 0.00035691, batch_accuracy: 1e+02
[35-5400] loss: 0.0010628, batch_accuracy: 1e+02
[35-5450] loss: 0.00029384, batch_accuracy: 1e+02
End of Epoch 35/100 -> loss: 0.58556, test accuracy: 87.3 - best accuracy: 88.3
[36-5500] loss: 0.00014004, batch_accuracy: 1e+02
[36-5550] loss: 0.0001613, batch_accuracy: 1e+02
[36-5600] loss: 2.8553e-05, batch_accuracy: 1e+02
[36-5650] loss: 0.00046541, batch_accuracy: 1e+02
End of Epoch 36/100 -> loss: 0.62339, test accuracy: 86.0 - best accuracy: 88.3
[37-5700] loss: 6.4366e-05, batch_accuracy: 1e+02
[37-5750] loss: 0.00013292, batch_accuracy: 1e+02
[37-5800] loss: 0.0066805, batch_accuracy: 1e+02
End of Epoch 37/100 -> loss: 0.60773, test accuracy: 86.7 - best accuracy: 88.3
[38-5850] loss: 0.00010702, batch_accuracy: 1e+02
[38-5900] loss: 9.4365e-05, batch_accuracy: 1e+02
[38-5950] loss: 0.00036535, batch_accuracy: 1e+02
End of Epoch 38/100 -> loss: 0.60558, test accuracy: 86.7 - best accuracy: 88.3
[39-6000] loss: 0.0007572, batch_accuracy: 1e+02
[39-6050] loss: 0.00024246, batch_accuracy: 1e+02
[39-6100] loss: 0.00033057, batch_accuracy: 1e+02
End of Epoch 39/100 -> loss: 0.57861, test accuracy: 86.7 - best accuracy: 88.3
[40-6150] loss: 8.5141e-05, batch_accuracy: 1e+02
[40-6200] loss: 0.00024113, batch_accuracy: 1e+02
[40-6250] loss: 0.00010836, batch_accuracy: 1e+02
End of Epoch 40/100 -> loss: 0.58079, test accuracy: 87.4 - best accuracy: 88.3
[41-6300] loss: 2.1666e-05, batch_accuracy: 1e+02
[41-6350] loss: 7.9828e-05, batch_accuracy: 1e+02
[41-6400] loss: 3.691e-05, batch_accuracy: 1e+02
End of Epoch 41/100 -> loss: 0.58277, test accuracy: 87.2 - best accuracy: 88.3
[42-6450] loss: 0.00014712, batch_accuracy: 1e+02
[42-6500] loss: 7.8619e-05, batch_accuracy: 1e+02
[42-6550] loss: 0.00030805, batch_accuracy: 1e+02
End of Epoch 42/100 -> loss: 0.59597, test accuracy: 86.9 - best accuracy: 88.3
[43-6600] loss: 6.157e-05, batch_accuracy: 1e+02
[43-6650] loss: 5.2198e-05, batch_accuracy: 1e+02
[43-6700] loss: 0.00011921, batch_accuracy: 1e+02
[43-6750] loss: 6.6643e-06, batch_accuracy: 1e+02
End of Epoch 43/100 -> loss: 0.59641, test accuracy: 87.0 - best accuracy: 88.3
[44-6800] loss: 0.0001534, batch_accuracy: 1e+02
[44-6850] loss: 9.753e-05, batch_accuracy: 1e+02
[44-6900] loss: 1.8986e-05, batch_accuracy: 1e+02
End of Epoch 44/100 -> loss: 0.58655, test accuracy: 87.3 - best accuracy: 88.3
[45-6950] loss: 6.2828e-05, batch_accuracy: 1e+02
[45-7000] loss: 1.8361e-05, batch_accuracy: 1e+02
[45-7050] loss: 3.4611e-05, batch_accuracy: 1e+02
End of Epoch 45/100 -> loss: 0.58626, test accuracy: 87.6 - best accuracy: 88.3
[46-7100] loss: 0.00024489, batch_accuracy: 1e+02
[46-7150] loss: 2.9695e-05, batch_accuracy: 1e+02
[46-7200] loss: 2.8805e-05, batch_accuracy: 1e+02
End of Epoch 46/100 -> loss: 0.5868, test accuracy: 87.7 - best accuracy: 88.3
[47-7250] loss: 4.52e-05, batch_accuracy: 1e+02
[47-7300] loss: 1.2122e-05, batch_accuracy: 1e+02
[47-7350] loss: 5.1445e-06, batch_accuracy: 1e+02
End of Epoch 47/100 -> loss: 0.57816, test accuracy: 87.5 - best accuracy: 88.3
[48-7400] loss: 0.007227, batch_accuracy: 1e+02
[48-7450] loss: 0.076734, batch_accuracy: 96.9
[48-7500] loss: 0.023738, batch_accuracy: 1e+02
End of Epoch 48/100 -> loss: 3.0727, test accuracy: 73.0 - best accuracy: 88.3
[49-7550] loss: 0.0057308, batch_accuracy: 1e+02
[49-7600] loss: 0.0729, batch_accuracy: 96.9
[49-7650] loss: 0.185, batch_accuracy: 87.5
End of Epoch 49/100 -> loss: 2.718, test accuracy: 76.1 - best accuracy: 88.3
[50-7700] loss: 0.053376, batch_accuracy: 1e+02
[50-7750] loss: 0.00090814, batch_accuracy: 1e+02
[50-7800] loss: 0.0020519, batch_accuracy: 1e+02
[50-7850] loss: 0.087861, batch_accuracy: 87.5
End of Epoch 50/100 -> loss: 0.78877, test accuracy: 81.6 - best accuracy: 88.3
[51-7900] loss: 0.0017798, batch_accuracy: 1e+02
[51-7950] loss: 0.22809, batch_accuracy: 96.9
[51-8000] loss: 0.15407, batch_accuracy: 96.9
End of Epoch 51/100 -> loss: 1.4392, test accuracy: 77.8 - best accuracy: 88.3
[52-8050] loss: 0.16332, batch_accuracy: 96.9
[52-8100] loss: 0.077424, batch_accuracy: 96.9
[52-8150] loss: 0.010698, batch_accuracy: 1e+02
End of Epoch 52/100 -> loss: 0.71604, test accuracy: 83.1 - best accuracy: 88.3
[53-8200] loss: 0.0041676, batch_accuracy: 1e+02
[53-8250] loss: 0.018385, batch_accuracy: 1e+02
[53-8300] loss: 0.025333, batch_accuracy: 96.9
End of Epoch 53/100 -> loss: 0.74429, test accuracy: 83.5 - best accuracy: 88.3
[54-8350] loss: 0.0021239, batch_accuracy: 1e+02
[54-8400] loss: 0.0006246, batch_accuracy: 1e+02
[54-8450] loss: 0.0015241, batch_accuracy: 1e+02
End of Epoch 54/100 -> loss: 0.58714, test accuracy: 86.4 - best accuracy: 88.3
[55-8500] loss: 0.011118, batch_accuracy: 1e+02
[55-8550] loss: 0.0068855, batch_accuracy: 1e+02
[55-8600] loss: 0.00039368, batch_accuracy: 1e+02
End of Epoch 55/100 -> loss: 0.61744, test accuracy: 85.5 - best accuracy: 88.3
[56-8650] loss: 0.00076348, batch_accuracy: 1e+02
[56-8700] loss: 0.00011978, batch_accuracy: 1e+02
[56-8750] loss: 0.00022873, batch_accuracy: 1e+02
End of Epoch 56/100 -> loss: 0.58157, test accuracy: 86.3 - best accuracy: 88.3
[57-8800] loss: 0.0003647, batch_accuracy: 1e+02
[57-8850] loss: 8.1712e-05, batch_accuracy: 1e+02
[57-8900] loss: 0.00021736, batch_accuracy: 1e+02
End of Epoch 57/100 -> loss: 0.57008, test accuracy: 86.9 - best accuracy: 88.3
[58-8950] loss: 0.0014667, batch_accuracy: 1e+02
[58-9000] loss: 0.0034627, batch_accuracy: 1e+02
[58-9050] loss: 0.00025395, batch_accuracy: 1e+02
[58-9100] loss: 0.0011444, batch_accuracy: 1e+02
End of Epoch 58/100 -> loss: 0.57082, test accuracy: 87.3 - best accuracy: 88.3
[59-9150] loss: 7.5055e-05, batch_accuracy: 1e+02
[59-9200] loss: 0.00011621, batch_accuracy: 1e+02
[59-9250] loss: 3.8681e-05, batch_accuracy: 1e+02
End of Epoch 59/100 -> loss: 0.57481, test accuracy: 86.7 - best accuracy: 88.3
[60-9300] loss: 2.8965e-05, batch_accuracy: 1e+02
[60-9350] loss: 5.6676e-05, batch_accuracy: 1e+02
[60-9400] loss: 0.00030648, batch_accuracy: 1e+02
End of Epoch 60/100 -> loss: 0.57703, test accuracy: 86.9 - best accuracy: 88.3
[61-9450] loss: 0.00010277, batch_accuracy: 1e+02
[61-9500] loss: 0.00074349, batch_accuracy: 1e+02
[61-9550] loss: 0.0001843, batch_accuracy: 1e+02
End of Epoch 61/100 -> loss: 0.5726, test accuracy: 87.1 - best accuracy: 88.3
[62-9600] loss: 0.00011966, batch_accuracy: 1e+02
[62-9650] loss: 1.6464e-05, batch_accuracy: 1e+02
[62-9700] loss: 0.00015275, batch_accuracy: 1e+02
End of Epoch 62/100 -> loss: 0.57727, test accuracy: 87.1 - best accuracy: 88.3
[63-9750] loss: 7.0096e-05, batch_accuracy: 1e+02
[63-9800] loss: 4.1919e-05, batch_accuracy: 1e+02
[63-9850] loss: 1.5738e-05, batch_accuracy: 1e+02
End of Epoch 63/100 -> loss: 0.57612, test accuracy: 87.1 - best accuracy: 88.3
[64-9900] loss: 5.7692e-05, batch_accuracy: 1e+02
[64-9950] loss: 0.00026749, batch_accuracy: 1e+02
[64-10000] loss: 0.00072917, batch_accuracy: 1e+02
End of Epoch 64/100 -> loss: 0.5778, test accuracy: 87.2 - best accuracy: 88.3
[65-10050] loss: 0.00011384, batch_accuracy: 1e+02
[65-10100] loss: 2.9851e-05, batch_accuracy: 1e+02
[65-10150] loss: 5.7177e-05, batch_accuracy: 1e+02
[65-10200] loss: 0.00016728, batch_accuracy: 1e+02
End of Epoch 65/100 -> loss: 0.57563, test accuracy: 87.1 - best accuracy: 88.3
[66-10250] loss: 4.9562e-05, batch_accuracy: 1e+02
[66-10300] loss: 1.0989e-05, batch_accuracy: 1e+02
[66-10350] loss: 1.2151e-05, batch_accuracy: 1e+02
End of Epoch 66/100 -> loss: 0.58101, test accuracy: 87.5 - best accuracy: 88.3
[67-10400] loss: 1.9687e-05, batch_accuracy: 1e+02
[67-10450] loss: 6.3014e-05, batch_accuracy: 1e+02
[67-10500] loss: 5.3765e-05, batch_accuracy: 1e+02
End of Epoch 67/100 -> loss: 0.58097, test accuracy: 87.6 - best accuracy: 88.3
[68-10550] loss: 2.8726e-05, batch_accuracy: 1e+02
[68-10600] loss: 9.2071e-05, batch_accuracy: 1e+02
[68-10650] loss: 8.4524e-06, batch_accuracy: 1e+02
End of Epoch 68/100 -> loss: 0.58371, test accuracy: 87.8 - best accuracy: 88.3
[69-10700] loss: 3.8258e-06, batch_accuracy: 1e+02
[69-10750] loss: 3.3005e-06, batch_accuracy: 1e+02
[69-10800] loss: 0.0001451, batch_accuracy: 1e+02
End of Epoch 69/100 -> loss: 0.5814, test accuracy: 87.9 - best accuracy: 88.3
[70-10850] loss: 4.9135e-06, batch_accuracy: 1e+02
[70-10900] loss: 6.9474e-06, batch_accuracy: 1e+02
[70-10950] loss: 1.957e-05, batch_accuracy: 1e+02
End of Epoch 70/100 -> loss: 0.58487, test accuracy: 87.9 - best accuracy: 88.3
[71-11000] loss: 1.1093e-05, batch_accuracy: 1e+02
[71-11050] loss: 2.876e-05, batch_accuracy: 1e+02
[71-11100] loss: 0.00017237, batch_accuracy: 1e+02
End of Epoch 71/100 -> loss: 0.58554, test accuracy: 87.7 - best accuracy: 88.3
[72-11150] loss: 8.8621e-06, batch_accuracy: 1e+02
[72-11200] loss: 4.623e-06, batch_accuracy: 1e+02
[72-11250] loss: 1.5307e-05, batch_accuracy: 1e+02
[72-11300] loss: 7.9009e-06, batch_accuracy: 1e+02
End of Epoch 72/100 -> loss: 0.58573, test accuracy: 87.7 - best accuracy: 88.3
[73-11350] loss: 1.1548e-06, batch_accuracy: 1e+02
[73-11400] loss: 6.7947e-06, batch_accuracy: 1e+02
[73-11450] loss: 2.7549e-05, batch_accuracy: 1e+02
End of Epoch 73/100 -> loss: 0.58707, test accuracy: 87.5 - best accuracy: 88.3
[74-11500] loss: 6.6532e-06, batch_accuracy: 1e+02
[74-11550] loss: 2.8007e-05, batch_accuracy: 1e+02
[74-11600] loss: 1.3232e-05, batch_accuracy: 1e+02
End of Epoch 74/100 -> loss: 0.57954, test accuracy: 88.3 - best accuracy: 88.3
[75-11650] loss: 0.013881, batch_accuracy: 1e+02
[75-11700] loss: 0.10686, batch_accuracy: 96.9
[75-11750] loss: 0.16434, batch_accuracy: 96.9
End of Epoch 75/100 -> loss: 6.2643, test accuracy: 56.4 - best accuracy: 88.3
[76-11800] loss: 0.12258, batch_accuracy: 93.8
[76-11850] loss: 0.57097, batch_accuracy: 93.8
[76-11900] loss: 0.049826, batch_accuracy: 1e+02
End of Epoch 76/100 -> loss: 1.1509, test accuracy: 75.5 - best accuracy: 88.3
[77-11950] loss: 0.0063309, batch_accuracy: 1e+02
[77-12000] loss: 0.033451, batch_accuracy: 96.9
[77-12050] loss: 0.0020035, batch_accuracy: 1e+02
End of Epoch 77/100 -> loss: 0.8774, test accuracy: 80.7 - best accuracy: 88.3
[78-12100] loss: 0.003179, batch_accuracy: 1e+02
[78-12150] loss: 0.2932, batch_accuracy: 93.8
[78-12200] loss: 0.003725, batch_accuracy: 1e+02
End of Epoch 78/100 -> loss: 0.80633, test accuracy: 82.3 - best accuracy: 88.3
[79-12250] loss: 0.088071, batch_accuracy: 96.9
[79-12300] loss: 0.0085806, batch_accuracy: 1e+02
[79-12350] loss: 0.0045837, batch_accuracy: 1e+02
[79-12400] loss: 0.00095528, batch_accuracy: 1e+02
End of Epoch 79/100 -> loss: 0.74874, test accuracy: 82.0 - best accuracy: 88.3
[80-12450] loss: 0.00071641, batch_accuracy: 1e+02
[80-12500] loss: 0.00010282, batch_accuracy: 1e+02
[80-12550] loss: 0.00084829, batch_accuracy: 1e+02
End of Epoch 80/100 -> loss: 0.79107, test accuracy: 82.0 - best accuracy: 88.3
[81-12600] loss: 0.0051606, batch_accuracy: 1e+02
[81-12650] loss: 0.085301, batch_accuracy: 93.8
[81-12700] loss: 0.012851, batch_accuracy: 1e+02
End of Epoch 81/100 -> loss: 0.90386, test accuracy: 79.9 - best accuracy: 88.3
[82-12750] loss: 0.0029287, batch_accuracy: 1e+02
[82-12800] loss: 0.00013303, batch_accuracy: 1e+02
[82-12850] loss: 0.001276, batch_accuracy: 1e+02
End of Epoch 82/100 -> loss: 0.70592, test accuracy: 84.1 - best accuracy: 88.3
[83-12900] loss: 0.00028289, batch_accuracy: 1e+02
[83-12950] loss: 0.00045542, batch_accuracy: 1e+02
[83-13000] loss: 0.00035251, batch_accuracy: 1e+02
End of Epoch 83/100 -> loss: 0.7306, test accuracy: 84.3 - best accuracy: 88.3
[84-13050] loss: 0.00043806, batch_accuracy: 1e+02
[84-13100] loss: 0.00011244, batch_accuracy: 1e+02
[84-13150] loss: 2.8178e-05, batch_accuracy: 1e+02
End of Epoch 84/100 -> loss: 0.77932, test accuracy: 84.0 - best accuracy: 88.3
[85-13200] loss: 0.0011404, batch_accuracy: 1e+02
[85-13250] loss: 0.00017412, batch_accuracy: 1e+02
[85-13300] loss: 0.0037707, batch_accuracy: 1e+02
End of Epoch 85/100 -> loss: 0.93506, test accuracy: 82.0 - best accuracy: 88.3
[86-13350] loss: 0.00079819, batch_accuracy: 1e+02
[86-13400] loss: 0.0016098, batch_accuracy: 1e+02
[86-13450] loss: 0.01379, batch_accuracy: 1e+02
[86-13500] loss: 0.00080581, batch_accuracy: 1e+02
End of Epoch 86/100 -> loss: 0.74859, test accuracy: 83.7 - best accuracy: 88.3
[87-13550] loss: 0.00014159, batch_accuracy: 1e+02
[87-13600] loss: 0.022126, batch_accuracy: 1e+02
[87-13650] loss: 0.0007214, batch_accuracy: 1e+02
End of Epoch 87/100 -> loss: 0.82437, test accuracy: 82.8 - best accuracy: 88.3
[88-13700] loss: 0.00031081, batch_accuracy: 1e+02
[88-13750] loss: 0.00025751, batch_accuracy: 1e+02
[88-13800] loss: 0.00020505, batch_accuracy: 1e+02
End of Epoch 88/100 -> loss: 0.74837, test accuracy: 83.9 - best accuracy: 88.3
[89-13850] loss: 0.00026881, batch_accuracy: 1e+02
[89-13900] loss: 0.0010069, batch_accuracy: 1e+02
[89-13950] loss: 4.0654e-05, batch_accuracy: 1e+02
End of Epoch 89/100 -> loss: 0.71768, test accuracy: 84.9 - best accuracy: 88.3
[90-14000] loss: 5.5506e-05, batch_accuracy: 1e+02
[90-14050] loss: 0.00037507, batch_accuracy: 1e+02
[90-14100] loss: 3.9302e-05, batch_accuracy: 1e+02
End of Epoch 90/100 -> loss: 0.69985, test accuracy: 85.1 - best accuracy: 88.3
[91-14150] loss: 0.00021256, batch_accuracy: 1e+02
[91-14200] loss: 0.00016279, batch_accuracy: 1e+02
[91-14250] loss: 0.00036807, batch_accuracy: 1e+02
End of Epoch 91/100 -> loss: 0.70423, test accuracy: 85.0 - best accuracy: 88.3
[92-14300] loss: 7.8934e-05, batch_accuracy: 1e+02
[92-14350] loss: 0.00022027, batch_accuracy: 1e+02
[92-14400] loss: 0.002423, batch_accuracy: 1e+02
End of Epoch 92/100 -> loss: 0.8334, test accuracy: 82.8 - best accuracy: 88.3
[93-14450] loss: 0.00061748, batch_accuracy: 1e+02
[93-14500] loss: 0.025021, batch_accuracy: 96.9
[93-14550] loss: 0.035677, batch_accuracy: 1e+02
[93-14600] loss: 0.079671, batch_accuracy: 96.9
End of Epoch 93/100 -> loss: 4.3432, test accuracy: 70.2 - best accuracy: 88.3
[94-14650] loss: 0.013446, batch_accuracy: 1e+02
[94-14700] loss: 0.008851, batch_accuracy: 1e+02
[94-14750] loss: 0.012837, batch_accuracy: 1e+02
End of Epoch 94/100 -> loss: 1.4868, test accuracy: 78.0 - best accuracy: 88.3
[95-14800] loss: 0.022551, batch_accuracy: 1e+02
[95-14850] loss: 0.051763, batch_accuracy: 96.9
[95-14900] loss: 0.0070813, batch_accuracy: 1e+02
End of Epoch 95/100 -> loss: 0.76604, test accuracy: 82.5 - best accuracy: 88.3
[96-14950] loss: 0.00037777, batch_accuracy: 1e+02
[96-15000] loss: 0.0008721, batch_accuracy: 1e+02
[96-15050] loss: 0.0084796, batch_accuracy: 1e+02
End of Epoch 96/100 -> loss: 0.70851, test accuracy: 83.1 - best accuracy: 88.3
[97-15100] loss: 0.00031157, batch_accuracy: 1e+02
[97-15150] loss: 0.0013247, batch_accuracy: 1e+02
[97-15200] loss: 0.0005337, batch_accuracy: 1e+02
End of Epoch 97/100 -> loss: 0.69373, test accuracy: 83.9 - best accuracy: 88.3
[98-15250] loss: 0.00022284, batch_accuracy: 1e+02
[98-15300] loss: 0.00071659, batch_accuracy: 1e+02
[98-15350] loss: 0.00023535, batch_accuracy: 1e+02
End of Epoch 98/100 -> loss: 0.70465, test accuracy: 83.9 - best accuracy: 88.3
[99-15400] loss: 0.00016574, batch_accuracy: 1e+02
[99-15450] loss: 0.00025543, batch_accuracy: 1e+02
[99-15500] loss: 0.0002386, batch_accuracy: 1e+02
End of Epoch 99/100 -> loss: 0.7083, test accuracy: 84.3 - best accuracy: 88.3
[100-15550] loss: 2.5448e-05, batch_accuracy: 1e+02
[100-15600] loss: 0.00015178, batch_accuracy: 1e+02
[100-15650] loss: 9.9608e-06, batch_accuracy: 1e+02
[100-15700] loss: 0.0028883, batch_accuracy: 1e+02
End of Epoch 100/100 -> loss: 0.69954, test accuracy: 84.5 - best accuracy: 88.3

• 2021 @ GabrieleGhisleni
• GG