Randomized Hyperparameter Search in Tensorflow

Even a vanilla convolutional neural network has many hyperparamters, including those that define the overall architecture, choice of activation functions and learning rate. A standard grid search would require a huge amount of processing, given that a single network can take long time to train. An alternative is a randomized hyperparameter search.

Below, Tensorflow is used to build a CNN to classify the images in the CIFAR-10 dataset.

The optimal architecture, learning rate and other hyperparameters of the CNN are searched for using a randomized method.

The CIFAR-10 comes from: Learning Multiple Layers of Features from Tiny Images, Alex Krizhevsky, 2009

Data Preparation

# Provided on https://www.cs.toronto.edu/~kriz/cifar.html 
# Given a "pickled" file, returns a dictionary containing the image data
def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

The training data is stored in four files, which are read and merged.

import numpy as np

for file_no in range(4):
    # Obtain data dictionary from each file
    filename = "cifar-10-batches-py/data_batch_" + str(file_no + 1)
    image_batch = unpickle(filename)
    # First file, create numpy arrays containing data & labels
    if file_no == 0:
        # Reshape to 32x32 image with 3 channels (RGB), which is made to be the last axes
        image_data = image_batch[b'data'].reshape((-1,3,32,32)).transpose((0,2,3,1))
        image_labels = image_batch[b'labels']
    else: # Concatenate to one file
        new_data = image_batch[b'data'].reshape((-1,3,32,32)).transpose((0,2,3,1))
        image_data = np.concatenate([image_data, new_data])
        image_labels = np.concatenate([image_labels, image_batch[b'labels']])

print("Training data shape: ",image_data.shape)

# Create numpy array containing test data
test_batch = unpickle("cifar-10-batches-py/test_batch")
test_data = test_batch[b'data'].reshape((-1,3,32,32)).transpose((0,2,3,1))
test_labels = test_batch[b'labels']

# Obtain label names from the meta data
label_names = unpickle("cifar-10-batches-py/batches.meta")[b'label_names']
label_names = [l.decode('UTF-8') for l in label_names]

Training data shape:  (40000, 32, 32, 3)

Image samples

import matplotlib.pyplot as plt

fig, axes = plt.subplots(10,9,figsize=(7,9))
plt.subplots_adjust(wspace=0.01, hspace=0.01)

for image_type in range(10):
    # Obtain indices of images in each group
    group = np.argwhere(image_labels == image_type)
    # Display 1st 5 images from each group
    for col in range(8):
        n = group[col][0]
        img_rgb = image_data[n]
        axes[image_type,col].imshow(img_rgb)
        axes[image_type,col].axis('off')
    # Display image labels    
    axes[image_type,8].axis('off')    
    axes[image_type,8].text(0.1,0.5,label_names[image_type], fontsize=12)

plt.show()

Model Building

Hyperparameters

import tensorflow as tf
# Fixed hyperparameters
height = 32
width = 32
channels = 3
outputs = 10

n_epochs = 500
# number of epochs to try to get a lower loss before stopping
early_stop_rounds = 4

# number of iterations, within epoch, to do an accuracy check
acc_check = 25

# Variable Hyperparameters
hyperparam_range = {'filters1':[16, 32, 64, 96],
                    'ksize1':[3, 4, 5],
                    'filters2':[48, 64, 96, 128],
                    'ksize2':[3, 4, 5],
                    'filters3':[64, 96, 128],
                    'ksize3':[3, 4, 5],
                    'full_hidd1':[60, 100, 125],
                    'full_hidd2':[80, 100, 125],
                    'activation':['relu', 'lrelu', 'elu'],
                    'learning_rate':[0.001, 0.0015, 0.002, 0.003],
                    'batch_size':[32, 64, 128, 256],
                    'momentum':[0.9, 0.95, 0.99],
                    'patch_reduction':[0,1,2],
                    'optimizer':['adam', 'rmsprop', 'nesterov']
                   }

# Calculate the number of hyperparamter grid points
first_item = True
for key,values in hyperparam_range.items():
    count = len(values)
    if first_item:
        display = str(count)
        total = count
        first_item = False
    else:
        display = display + ' X ' + str(count)
        total *= count

print('The total number of possible hyperparameter combinations is ' + display + ' = ' + "{:,}".format(total))

The total number of possible hyperparameter combinations is 3 X 4 X 3 X 3 X 4 X 3 X 4 X 3 X 3 X 3 X 3 X 3 X 3 X 4 = 15,116,544

Convolutional Layer

def conv_layer(tensor_input, layer_no, filters, ksize, kstride, activation_unit, momentum, phase_train):
    # convolutional layer with batch normalisation and max pooling
    with tf.name_scope("conv_layer" + str(layer_no)):
        conv = tf.layers.conv2d(
            tensor_input,
            filters=filters,
            kernel_size=ksize,
            strides=[kstride,kstride],
            padding="SAME",
            activation=None
        )

        conv_bn = tf.layers.batch_normalization(
            inputs=conv,
            axis=-1,
            momentum=0.9,
            epsilon=0.001,
            center=True,
            scale=True,
            trainable=True,
            training = phase_train
        )

        #apply activation unit
        conv_bn_relu =  activation_unit(conv_bn)

        max_pool = tf.nn.max_pool(
            conv_bn_relu,
            ksize=[1,2,2,1],
            strides=[1,2,2,1],
            padding="VALID"
        )
            
        return max_pool

Graph defintion

def build_graph(hyperparam):
    # Retrieve hyperparamaters from dictionary   
    filters1 = hyperparam['filters1']
    ksize1 = hyperparam['ksize1']
    filters2 = hyperparam['filters2']
    ksize2 = hyperparam['ksize2']
    filters3 = hyperparam['filters3']
    ksize3 = hyperparam['ksize3']
    full_hidd1 = hyperparam['full_hidd1']
    full_hidd2 = hyperparam['full_hidd2']
    activation = hyperparam['activation']
    learning_rate = hyperparam['learning_rate']
    momentum = hyperparam['momentum']
    patch_reduction =  hyperparam['patch_reduction']
    optimizer_method = hyperparam['optimizer']
    
    patch_height = height - 2 * patch_reduction
    patch_width = width - 2 * patch_reduction
    
    if activation == 'elu':
        activation_unit = tf.nn.elu
    elif activation == 'lrelu':
        activation_unit = tf.nn.leaky_relu
    else:    
        activation_unit = tf.nn.relu
        
    graph = tf.Graph()
    with graph.as_default():
        X = tf.placeholder(shape=(None, patch_height, patch_width, channels), dtype=tf.float32)
        y = tf.placeholder(shape=(None), dtype=tf.int32)
        phase_train = tf.placeholder(tf.bool, name='phase_train')

        # 1st convolutional layer
        conv1 = conv_layer(X, 1, filters1, ksize1, 2, activation_unit, momentum, phase_train)
        
        # 2nd convolutional layer
        conv2 = conv_layer(conv1, 2, filters2, ksize2, 1, activation_unit, momentum, phase_train)
        
        # 3rd cnn
        conv3 = conv_layer(conv2, 3, filters3, ksize3, 1, activation_unit, momentum, phase_train)
        
        # 1st fully connected (dense) layer
        fully_conn1 = tf.layers.dense(conv3, full_hidd1, name="fully_conn1", activation=activation_unit)
        flat = tf.contrib.layers.flatten(fully_conn1)
        
        # 2nd fully connected layer
        fully_conn2 = tf.layers.dense(flat, full_hidd2, name="fully_conn2", activation=activation_unit)
        
        # Output layer (no activation function, as this is built into cross-entropy)
        logits = tf.layers.dense(fully_conn2, outputs, name="logits")

        # Cross Entropy Loss
        with tf.name_scope("loss"):
            xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
            loss = tf.reduce_mean(xentropy, name="loss")

        # Training    
        with tf.name_scope("train"):
            if optimizer_method == 'rmsprop':
                    optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate, momentum=0.9)
            elif optimizer_method == 'nesterov':
                    optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.9, use_nesterov = True)
            else:    
                optimizer = tf.train.AdamOptimizer(learning_rate)
            training_op = optimizer.minimize(loss)
    
        # Initialization & Saver
        init = tf.global_variables_initializer()
        saver = tf.train.Saver() 
        
        return graph, X, y, phase_train, logits, loss, training_op, init, saver

Model Training

# Function to train model, returns minimised loss
def train_model(batch_size = 128):
    training_size = image_data.shape[0]
    no_batches = training_size // batch_size
    
    with graph.as_default():
        # Ensure batch normalisation gets updated
        extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)    
        # Add Evaluation metrics
        with tf.name_scope("eval"):
            correct = tf.nn.in_top_k(logits, y, 1)
            accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
            acc_summary = tf.summary.scalar('Accuracy', accuracy)
            loss_summary = tf.summary.scalar('Loss', loss)
            
    # Initialise best loss for early stopping
    best_loss = 1e9
    best_acc = 1e9
    early_stopping = False
    
    # Run training
    with tf.Session(graph=graph) as sess:                     
        init.run()
        epoch = 0
        early_stop_count = 0
        # Initialise training accuracy sum
        sum_acc_train = 0.0
        sum_loss_train = 0.0
        count_train = 0
        while (epoch < n_epochs) and (not early_stopping):
            epoch += 1
            # Shuffle training data
            p = np.random.permutation(training_size)
            image_data_shuffle = image_data_patch[p]
            image_labels_shuffle = image_labels[p]
            for iteration in range(no_batches-1):
                # Train on mini-batch
                X_batch = image_data_shuffle[iteration * batch_size:(iteration + 1) * batch_size + 1]
                y_batch = image_labels_shuffle[iteration * batch_size:(iteration + 1) * batch_size + 1]
                sess.run([training_op, extra_update_ops], feed_dict={X: X_batch, y: y_batch, phase_train: True})
                # Evaluate model on current mini-batch
                if (iteration % acc_check == 0):
                    acc_train, loss_train = sess.run([accuracy, loss], feed_dict={X: X_batch,
                                                                                   y: y_batch,
                                                                                   phase_train: True})
                    # Update training average for current epoch
                    sum_acc_train += acc_train
                    sum_loss_train += loss_train
                    count_train += 1
                    mean_acc_train = sum_acc_train / count_train
                    mean_loss_train = sum_loss_train / count_train
                    print(epoch, iteration, "Train accuracy:", mean_acc_train, " Train loss:", mean_loss_train, end='\r')
                     
            # Print mean Train accuracy        
            print(epoch, iteration, "Train accuracy:", mean_acc_train, " Train loss:", mean_loss_train)
            train_summary = tf.Summary()
            train_summary.value.add(tag='eval/Accuracy', simple_value=mean_acc_train)
            train_summary.value.add(tag='eval/Loss', simple_value=mean_loss_train)
            file_writer_train.add_summary(train_summary, epoch)
            
            # Evaluate model on test data every epoch
            acc_test, loss_test, summary_str_acc_test, summary_str_loss_test = sess.run([accuracy, loss,
                                                                                         acc_summary,
                                                                                         loss_summary],
                                                                                        feed_dict={X: test_data_patch,
                                                                                                   y: test_labels,
                                                                                                   phase_train: False})
            # Check for mininimum loss
            if loss_test < best_loss:
                # Reset early stopping best loss and count
                best_loss = loss_test
                best_acc = acc_test
                best_train_acc = mean_acc_train
                best_train_loss = mean_loss_train
                early_stop_count = 0
                # Save model
                save_path = saver.save(sess, ckptfile)
            else:
                # Increment early stopping count
                early_stop_count += 1
            # Check if sufficient rounds (epochs) have passed without improvement
            if early_stop_count  > early_stop_rounds:
                # Flag early stopping, so training loop will stop
                early_stopping = True
                print("Early stopping, best loss: ", best_loss)
            else:    
                # Record and display test loss and accuracy
                file_writer_test.add_summary(summary_str_acc_test, epoch)
                file_writer_test.add_summary(summary_str_loss_test, epoch)
                print(epoch, "Test accuracy:", acc_test, " Test loss:", loss_test)
            
        # Return Best Loss obtaned and number of epochs
        return best_loss, best_acc, best_train_acc, best_train_loss, epoch

Random Hyperparameter Selection and Evaluation

import pandas as pd

import random
random.seed(261001)

import os.path
if not os.path.exists('./log'):
    os.makedirs('./log')
if not os.path.exists('./graphs'):
    os.makedirs('./graphs')
    
if(os.path.isfile('./log/model_hyperparam.csv')):
    hyperparam_df = pd.read_csv('./log/model_hyperparam.csv')
    new_results_file = False
else:
    new_results_file = True

# Random Model search
for n in range(50):
    # Create log directory using current timestamp
    from datetime import datetime
    now = datetime.now().strftime("%Y%m%dT%H%M%S")
    root_logdir = "./log"
    root_graphdir = "./graphs"
    logdir = "{}/run-{}/".format(root_logdir, now)
    graphdir = "{}/run-{}/".format(root_graphdir, now)
    #root_modeldir = "./model"
    graphfile = graphdir
    ckptfile = graphdir + "checkpoint.ckpt"

    # Note that the graph is not written using same filewriter as the logging data.
    # This allows the logs to be viewed during training before the filewriter is closed
    file_writer_train = tf.summary.FileWriter(logdir + '/train')
    file_writer_test = tf.summary.FileWriter(logdir + '/test')
    
    # tensorboard --logdir e:\Programming\TensorFlow\CIFAR-10\log
    # The command must be executed from same drive (E:) as the logdir

    tf.reset_default_graph()

    hyperparam_dict = {key:random.choice(values) for (key,values) in hyperparam_range.items()}
    print("Training Model with following hyperparameters:")
    print(hyperparam_dict)

    # Obtain graph and nodes required for training
    graph, X, y, phase_train, logits, loss, training_op, init, saver = build_graph(hyperparam = hyperparam_dict)
    
    # Write graph to file and close to avoid TensorBoard conflict
    file_writer_graph = tf.summary.FileWriter(graphfile, graph)
    file_writer_graph.close()

    # Extract image patches
    patch_reduction =  hyperparam_dict['patch_reduction']
    if patch_reduction == 0:
        image_data_patch = image_data
        test_data_patch = test_data
    else:
        image_data_patch = image_data[:,patch_reduction:-patch_reduction,patch_reduction:-patch_reduction,:]
        test_data_patch = test_data[:,patch_reduction:-patch_reduction,patch_reduction:-patch_reduction,:]
    
    # Train the model
    best_loss, best_acc, best_train_acc, best_train_loss, no_epochs = train_model(batch_size = hyperparam_dict['batch_size'])
    
    # Add results to dictionary
    hyperparam_dict['best_loss'] = best_loss
    hyperparam_dict['best_acc'] = best_acc
    hyperparam_dict['best_train_acc'] = best_train_acc
    hyperparam_dict['best_train_loss'] = best_train_loss
    hyperparam_dict['no_epochs'] = no_epochs
    hyperparam_dict['logdir'] = logdir
    
    # Add results to dataframe
    if new_results_file:
        hyperparam_df = pd.DataFrame(hyperparam_dict, index=[0])
        new_results_file = False
    else:
        hyperparam_df = hyperparam_df.append(hyperparam_dict, ignore_index=True)
        
    # Write results file (don't wait until end, as next model may be interrupted)
    hyperparam_df.to_csv('./log/model_hyperparam_2.csv', index=False)

    # Close filewriters
    file_writer_train.close()
    file_writer_test.close()
    
    # NOTE - From this point onwards there are only training logs.
    # Results and Conclusion are in a seperate notebook

Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.0015, 'ksize1': 3, 'filters1': 64, 'activation': 'relu', 'ksize3': 3, 'ksize2': 3, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.9, 'filters2': 96}
1 310 Train accuracy: 0.409063800023  Train loss: 1.68374873125
1 Test accuracy: 0.4797  Test loss: 1.42656
2 310 Train accuracy: 0.486285027403  Train loss: 1.46946371977
2 Test accuracy: 0.5191  Test loss: 1.33813
3 310 Train accuracy: 0.532697273371  Train loss: 1.33664505298
3 Test accuracy: 0.5597  Test loss: 1.23606
4 310 Train accuracy: 0.56305903139  Train loss: 1.245241848322
4 Test accuracy: 0.5943  Test loss: 1.15146
5 310 Train accuracy: 0.588789503849  Train loss: 1.17042976434
5 Test accuracy: 0.6128  Test loss: 1.09734
6 310 Train accuracy: 0.611409260294  Train loss: 1.10616910305
6 Test accuracy: 0.6045  Test loss: 1.12523
7 310 Train accuracy: 0.630888488594  Train loss: 1.05214448159
7 Test accuracy: 0.6223  Test loss: 1.10783
8 310 Train accuracy: 0.646466903675  Train loss: 1.00854980831
8 Test accuracy: 0.5528  Test loss: 1.36412
9 310 Train accuracy: 0.663420127243  Train loss: 0.964142434617
9 Test accuracy: 0.556  Test loss: 1.37328
10 310 Train accuracy: 0.678115681272  Train loss: 0.925783511079
Early stopping, best loss:  1.09734
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 64, 'learning_rate': 0.001, 'ksize1': 4, 'filters1': 32, 'activation': 'elu', 'ksize3': 3, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 96}
1 623 Train accuracy: 0.454153858423  Train loss: 1.52709571838
1 Test accuracy: 0.5084  Test loss: 1.35535
2 623 Train accuracy: 0.522461556196  Train loss: 1.32944801331
2 Test accuracy: 0.473  Test loss: 1.62322
3 623 Train accuracy: 0.570666685104  Train loss: 1.21257860104
3 Test accuracy: 0.5558  Test loss: 1.28646
4 623 Train accuracy: 0.604000018239  Train loss: 1.13424204946
4 Test accuracy: 0.6255  Test loss: 1.07537
5 623 Train accuracy: 0.622400018215  Train loss: 1.08027180481
5 Test accuracy: 0.6095  Test loss: 1.11579
6 623 Train accuracy: 0.640102581978  Train loss: 1.02869379878
6 Test accuracy: 0.6331  Test loss: 1.04492
7 623 Train accuracy: 0.655560457025  Train loss: 0.987630870002
7 Test accuracy: 0.6527  Test loss: 1.01085
8 623 Train accuracy: 0.671307709217  Train loss: 0.950404718222
8 Test accuracy: 0.6069  Test loss: 1.25032
9 623 Train accuracy: 0.685880358219  Train loss: 0.916311492125
9 Test accuracy: 0.6662  Test loss: 0.978574
10 623 Train accuracy: 0.698092323542  Train loss: 0.884779557824
10 Test accuracy: 0.6632  Test loss: 1.00062
11 623 Train accuracy: 0.708923092322  Train loss: 0.853931819851
11 Test accuracy: 0.6502  Test loss: 1.06671
12 623 Train accuracy: 0.719846168756  Train loss: 0.823981718719
12 Test accuracy: 0.6575  Test loss: 1.06249
13 623 Train accuracy: 0.728662736416  Train loss: 0.797341091083
13 Test accuracy: 0.6515  Test loss: 1.09549
14 623 Train accuracy: 0.739164849179  Train loss: 0.769009385705
Early stopping, best loss:  0.978574
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 16, 'activation': 'lrelu', 'ksize3': 5, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.99, 'filters2': 96}
1 310 Train accuracy: 0.47465712749  Train loss: 1.456836159417
1 Test accuracy: 0.4368  Test loss: 1.50571
2 310 Train accuracy: 0.55724508258  Train loss: 1.231833567996
2 Test accuracy: 0.5731  Test loss: 1.21186
3 310 Train accuracy: 0.604253626787  Train loss: 1.11085998553
3 Test accuracy: 0.6116  Test loss: 1.0982
4 310 Train accuracy: 0.636851519346  Train loss: 1.02280224057
4 Test accuracy: 0.6379  Test loss: 1.0393
5 310 Train accuracy: 0.665235537749  Train loss: 0.944229304332
5 Test accuracy: 0.6731  Test loss: 0.937059
6 310 Train accuracy: 0.689028024673  Train loss: 0.878086318954
6 Test accuracy: 0.6672  Test loss: 1.00873
7 310 Train accuracy: 0.711559756772  Train loss: 0.815225914939
7 Test accuracy: 0.671  Test loss: 1.0186
8 310 Train accuracy: 0.732930828172  Train loss: 0.759676974843
8 Test accuracy: 0.6322  Test loss: 1.27424
9 310 Train accuracy: 0.752534288117  Train loss: 0.707917561261
9 Test accuracy: 0.6708  Test loss: 1.18355
10 310 Train accuracy: 0.769051879644  Train loss: 0.662771021689
Early stopping, best loss:  0.937059
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 64, 'activation': 'relu', 'ksize3': 5, 'ksize2': 5, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 48}
1 1248 Train accuracy: 0.504242429733  Train loss: 1.39686472654
1 Test accuracy: 0.4961  Test loss: 1.40205
2 1248 Train accuracy: 0.564545457065  Train loss: 1.22508590579
2 Test accuracy: 0.5398  Test loss: 1.34031
3 1248 Train accuracy: 0.60949495395  Train loss: 1.104395800039
3 Test accuracy: 0.6453  Test loss: 1.01107
4 1248 Train accuracy: 0.63727273196  Train loss: 1.024953260871
4 Test accuracy: 0.5943  Test loss: 1.29981
5 1248 Train accuracy: 0.664969702244  Train loss: 0.949212236404
5 Test accuracy: 0.6593  Test loss: 1.00205
6 1248 Train accuracy: 0.685353541772  Train loss: 0.890941724877
6 Test accuracy: 0.6469  Test loss: 1.10088
7 1248 Train accuracy: 0.704069271258  Train loss: 0.840530431441
7 Test accuracy: 0.6966  Test loss: 0.940112
8 1225 Train accuracy: 0.721818189025  Train loss: 0.7932977758721248 Train accuracy: 0.721818189025  Train loss: 0.793297775872
8 Test accuracy: 0.6879  Test loss: 0.965718
9 1248 Train accuracy: 0.735353542699  Train loss: 0.755010371977
9 Test accuracy: 0.6955  Test loss: 0.946362
10 1248 Train accuracy: 0.748727280259  Train loss: 0.716451870948
10 Test accuracy: 0.6927  Test loss: 1.00483
11 1248 Train accuracy: 0.761763093363  Train loss: 0.680053448785
11 Test accuracy: 0.7022  Test loss: 1.01824
12 1248 Train accuracy: 0.772373745739  Train loss: 0.649308283304
Early stopping, best loss:  0.940112
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'rmsprop', 'batch_size': 64, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 64, 'activation': 'lrelu', 'ksize3': 4, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.9, 'filters2': 96}
1 600 Train accuracy: 0.4603077057  Train loss: 1.4728638362921623 Train accuracy: 0.4603077057  Train loss: 1.47286383629
1 Test accuracy: 0.528  Test loss: 1.32898
2 623 Train accuracy: 0.540615402609  Train loss: 1.27431922913
2 Test accuracy: 0.537  Test loss: 1.35324
3 623 Train accuracy: 0.58912822336  Train loss: 1.148206169613
3 Test accuracy: 0.5824  Test loss: 1.23284
4 623 Train accuracy: 0.62046155639  Train loss: 1.067476554513
4 Test accuracy: 0.5802  Test loss: 1.22046
5 623 Train accuracy: 0.645538478911  Train loss: 1.00054337597
5 Test accuracy: 0.5838  Test loss: 1.28439
6 623 Train accuracy: 0.66769232437  Train loss: 0.9447294129935
6 Test accuracy: 0.6785  Test loss: 0.969094
7 600 Train accuracy: 0.685186829269  Train loss: 0.894331106799623 Train accuracy: 0.685186829269  Train loss: 0.894331106799
7 Test accuracy: 0.6674  Test loss: 1.03059
8 600 Train accuracy: 0.701230784692  Train loss: 0.850290855616623 Train accuracy: 0.701230784692  Train loss: 0.850290855616
8 Test accuracy: 0.5955  Test loss: 1.46908
9 623 Train accuracy: 0.716581211388  Train loss: 0.809439736874
9 Test accuracy: 0.6904  Test loss: 1.02757
10 623 Train accuracy: 0.726461552888  Train loss: 0.779120973111
10 Test accuracy: 0.6715  Test loss: 1.07786
11 623 Train accuracy: 0.738629384501  Train loss: 0.747330323241
Early stopping, best loss:  0.969094
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 32, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 16, 'activation': 'lrelu', 'ksize3': 4, 'ksize2': 4, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.9, 'filters2': 128}
1 1248 Train accuracy: 0.517575765848  Train loss: 1.32563982017
1 Test accuracy: 0.4855  Test loss: 1.483
2 1248 Train accuracy: 0.586969704628  Train loss: 1.15039011184
2 Test accuracy: 0.6291  Test loss: 1.03717
3 1248 Train accuracy: 0.633939401309  Train loss: 1.02564093967
3 Test accuracy: 0.6398  Test loss: 1.03866
4 1248 Train accuracy: 0.673030311465  Train loss: 0.926152368337
4 Test accuracy: 0.6706  Test loss: 0.967186
5 1248 Train accuracy: 0.703272736073  Train loss: 0.844353604615
5 Test accuracy: 0.6778  Test loss: 1.00272
6 1248 Train accuracy: 0.730707079768  Train loss: 0.773266502718
6 Test accuracy: 0.6757  Test loss: 1.01409
7 1248 Train accuracy: 0.753852823292  Train loss: 0.712822854178
7 Test accuracy: 0.6931  Test loss: 1.01147
8 1248 Train accuracy: 0.775757585317  Train loss: 0.652634745333
8 Test accuracy: 0.6885  Test loss: 1.09825
9 1248 Train accuracy: 0.793872064087  Train loss: 0.603850086364
Early stopping, best loss:  0.967186
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 16, 'activation': 'relu', 'ksize3': 4, 'ksize2': 4, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 64}
1 310 Train accuracy: 0.489564703061  Train loss: 1.42583602208
1 Test accuracy: 0.5058  Test loss: 1.36474
2 310 Train accuracy: 0.54382826961  Train loss: 1.260449971148
2 Test accuracy: 0.6008  Test loss: 1.13916
3 310 Train accuracy: 0.588749754123  Train loss: 1.14234648454
3 Test accuracy: 0.6321  Test loss: 1.04616
4 310 Train accuracy: 0.622689327368  Train loss: 1.05598349067
4 Test accuracy: 0.635  Test loss: 1.05643
5 310 Train accuracy: 0.650447229239  Train loss: 0.983543625245
5 Test accuracy: 0.6554  Test loss: 1.01896
6 310 Train accuracy: 0.674021070584  Train loss: 0.923367462861
6 Test accuracy: 0.6278  Test loss: 1.09082
7 310 Train accuracy: 0.69460771896  Train loss: 0.8674884274775
7 Test accuracy: 0.674  Test loss: 0.9892
8 310 Train accuracy: 0.712805605852  Train loss: 0.821888229022
8 Test accuracy: 0.6577  Test loss: 1.06744
9 310 Train accuracy: 0.729344729684  Train loss: 0.777613452102
9 Test accuracy: 0.6624  Test loss: 1.08745
10 300 Train accuracy: 0.745080501758  Train loss: 0.735293491529310 Train accuracy: 0.745080501758  Train loss: 0.735293491529
10 Test accuracy: 0.6452  Test loss: 1.175
11 310 Train accuracy: 0.757304711359  Train loss: 0.699197584629
11 Test accuracy: 0.6547  Test loss: 1.25719
12 310 Train accuracy: 0.769777381649  Train loss: 0.664759965375
Early stopping, best loss:  0.9892
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 32, 'learning_rate': 0.002, 'ksize1': 3, 'filters1': 96, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 5, 'filters3': 96, 'patch_reduction': 0, 'momentum': 0.95, 'filters2': 48}
1 1248 Train accuracy: 0.537575763464  Train loss: 1.26713875771
1 Test accuracy: 0.5302  Test loss: 1.34406
2 1248 Train accuracy: 0.601212126315  Train loss: 1.09841675818
2 Test accuracy: 0.6041  Test loss: 1.16503
3 1248 Train accuracy: 0.640000005364  Train loss: 0.999986336827
3 Test accuracy: 0.6161  Test loss: 1.08606
4 1248 Train accuracy: 0.668030308634  Train loss: 0.931852890402
4 Test accuracy: 0.6387  Test loss: 1.09004
5 1248 Train accuracy: 0.692000006557  Train loss: 0.867231968045
5 Test accuracy: 0.6947  Test loss: 0.90603
6 1248 Train accuracy: 0.714040411015  Train loss: 0.810271756649
6 Test accuracy: 0.662  Test loss: 1.00591
7 1248 Train accuracy: 0.731082258139  Train loss: 0.765001005743
7 Test accuracy: 0.6935  Test loss: 0.96179
8 1248 Train accuracy: 0.748106068298  Train loss: 0.719913483188
8 Test accuracy: 0.6812  Test loss: 1.0225
9 1248 Train accuracy: 0.763232331475  Train loss: 0.678005452322
9 Test accuracy: 0.6942  Test loss: 1.03395
10 1248 Train accuracy: 0.777030312121  Train loss: 0.640022316158
Early stopping, best loss:  0.90603
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 32, 'learning_rate': 0.001, 'ksize1': 4, 'filters1': 96, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 5, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 64}
1 1248 Train accuracy: 0.517575764656  Train loss: 1.36621991754
1 Test accuracy: 0.5142  Test loss: 1.37517
2 1248 Train accuracy: 0.576969703436  Train loss: 1.20593243003
2 Test accuracy: 0.6066  Test loss: 1.10379
3 1248 Train accuracy: 0.622222228845  Train loss: 1.07026626209
3 Test accuracy: 0.6133  Test loss: 1.08191
4 1248 Train accuracy: 0.650303037763  Train loss: 0.996488863528 Train accuracy: 0.650303037763  Train loss: 0.996488863528
4 Test accuracy: 0.6011  Test loss: 1.17854
5 1248 Train accuracy: 0.678303038359  Train loss: 0.926169564486
5 Test accuracy: 0.6569  Test loss: 1.00884
6 1248 Train accuracy: 0.696060614189  Train loss: 0.872020863146
6 Test accuracy: 0.6831  Test loss: 0.923607
7 1248 Train accuracy: 0.714025982448  Train loss: 0.824741315288
7 Test accuracy: 0.6339  Test loss: 1.12997
8 1248 Train accuracy: 0.730909099132  Train loss: 0.782182721607
8 Test accuracy: 0.6314  Test loss: 1.13243
9 1248 Train accuracy: 0.747272735569  Train loss: 0.739550959898
9 Test accuracy: 0.6851  Test loss: 0.962814
10 1248 Train accuracy: 0.76242425108  Train loss: 0.7013517373861
10 Test accuracy: 0.698  Test loss: 0.937633
11 1248 Train accuracy: 0.775316813426  Train loss: 0.666559484059
Early stopping, best loss:  0.923607
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.0015, 'ksize1': 5, 'filters1': 96, 'activation': 'relu', 'ksize3': 4, 'ksize2': 5, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 128}
1 300 Train accuracy: 0.457960650898  Train loss: 1.53414009168310 Train accuracy: 0.457960650898  Train loss: 1.53414009168
1 Test accuracy: 0.5189  Test loss: 1.34408
2 310 Train accuracy: 0.532498515283  Train loss: 1.31920792506
2 Test accuracy: 0.5219  Test loss: 1.35945
3 310 Train accuracy: 0.577420000656  Train loss: 1.18540446728
3 Test accuracy: 0.5782  Test loss: 1.21607
4 310 Train accuracy: 0.61538461868  Train loss: 1.085357333614
4 Test accuracy: 0.5665  Test loss: 1.20897
5 310 Train accuracy: 0.646273110348  Train loss: 1.00605971722
5 Test accuracy: 0.5745  Test loss: 1.30936
6 310 Train accuracy: 0.674915526826  Train loss: 0.932064522535
6 Test accuracy: 0.6315  Test loss: 1.09599
7 310 Train accuracy: 0.699207771529  Train loss: 0.872210348045
7 Test accuracy: 0.6293  Test loss: 1.14345
8 310 Train accuracy: 0.719290401428  Train loss: 0.818556157442
8 Test accuracy: 0.6228  Test loss: 1.20838
9 310 Train accuracy: 0.737891739696  Train loss: 0.768972524466
9 Test accuracy: 0.6559  Test loss: 1.07677
10 310 Train accuracy: 0.756589149454  Train loss: 0.720669568387
10 Test accuracy: 0.6521  Test loss: 1.17237
11 310 Train accuracy: 0.773892774813  Train loss: 0.676206114513
11 Test accuracy: 0.6534  Test loss: 1.19274
12 310 Train accuracy: 0.789703835614  Train loss: 0.634909322915
12 Test accuracy: 0.6525  Test loss: 1.25576
13 310 Train accuracy: 0.80381633716  Train loss: 0.5963380988363
13 Test accuracy: 0.6539  Test loss: 1.31321
14 310 Train accuracy: 0.816636849522  Train loss: 0.561028954665
Early stopping, best loss:  1.07677
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.0015, 'ksize1': 5, 'filters1': 64, 'activation': 'elu', 'ksize3': 5, 'ksize2': 4, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 96}
1 310 Train accuracy: 0.476446037109  Train loss: 1.49312727268
1 Test accuracy: 0.5334  Test loss: 1.30542
2 310 Train accuracy: 0.555754326857  Train loss: 1.26215858872
2 Test accuracy: 0.5711  Test loss: 1.21867
3 310 Train accuracy: 0.601470883076  Train loss: 1.14037590608
3 Test accuracy: 0.6357  Test loss: 1.05067
4 310 Train accuracy: 0.637298749043  Train loss: 1.03990190763
4 Test accuracy: 0.5794  Test loss: 1.29995
5 310 Train accuracy: 0.66320811235  Train loss: 0.9679945698158
5 Test accuracy: 0.6445  Test loss: 1.0445
6 310 Train accuracy: 0.686841584169  Train loss: 0.904742130102
6 Test accuracy: 0.662  Test loss: 0.993874
7 310 Train accuracy: 0.707130081051  Train loss: 0.851402648858
7 Test accuracy: 0.632  Test loss: 1.10106
8 310 Train accuracy: 0.726222421687  Train loss: 0.802532788939
8 Test accuracy: 0.6514  Test loss: 1.04612
9 310 Train accuracy: 0.744649838688  Train loss: 0.754849540372
9 Test accuracy: 0.6451  Test loss: 1.13784
10 310 Train accuracy: 0.761121051587  Train loss: 0.711041486263
10 Test accuracy: 0.6346  Test loss: 1.195
11 300 Train accuracy: 0.776549034185  Train loss: 0.670937095489310 Train accuracy: 0.776549034185  Train loss: 0.670937095489
Early stopping, best loss:  0.993874
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 96, 'activation': 'relu', 'ksize3': 5, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 48}
1 310 Train accuracy: 0.504472269462  Train loss: 1.38822115843
1 Test accuracy: 0.4443  Test loss: 1.62943
2 300 Train accuracy: 0.577817534025  Train loss: 1.20451738972310 Train accuracy: 0.577817534025  Train loss: 1.20451738972
2 Test accuracy: 0.5151  Test loss: 1.39218
3 310 Train accuracy: 0.61498708297  Train loss: 1.094938449377
3 Test accuracy: 0.5978  Test loss: 1.17216
4 310 Train accuracy: 0.646094217896  Train loss: 1.00197953444
4 Test accuracy: 0.6315  Test loss: 1.06304
5 300 Train accuracy: 0.671794874852  Train loss: 0.937196888373310 Train accuracy: 0.671794874852  Train loss: 0.937196888373
5 Test accuracy: 0.6781  Test loss: 0.965303
6 310 Train accuracy: 0.692009542997  Train loss: 0.8802427966630.692009542997  Train loss: 0.880242796663
6 Test accuracy: 0.6755  Test loss: 0.988495
7 310 Train accuracy: 0.710196779979  Train loss: 0.829422209944
7 Test accuracy: 0.6724  Test loss: 1.0108
8 310 Train accuracy: 0.726967798976  Train loss: 0.781621958487
8 Test accuracy: 0.604  Test loss: 1.37207
9 310 Train accuracy: 0.741469554412  Train loss: 0.740189664384
9 Test accuracy: 0.6822  Test loss: 1.02933
10 310 Train accuracy: 0.754800237601  Train loss: 0.703563261491
Early stopping, best loss:  0.965303
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 256, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 16, 'activation': 'elu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 48}
1 154 Train accuracy: 0.397998872612  Train loss: 1.73074061532
1 Test accuracy: 0.4289  Test loss: 1.5865
2 154 Train accuracy: 0.467481914908  Train loss: 1.51243938719
2 Test accuracy: 0.5312  Test loss: 1.32296
3 154 Train accuracy: 0.518436147698  Train loss: 1.37870785736
3 Test accuracy: 0.4753  Test loss: 1.54018
4 154 Train accuracy: 0.548916043714  Train loss: 1.29001776235
4 Test accuracy: 0.5824  Test loss: 1.20795
5 154 Train accuracy: 0.571762069421  Train loss: 1.22930658204
5 Test accuracy: 0.5645  Test loss: 1.25797
6 154 Train accuracy: 0.586158956623  Train loss: 1.18305636588
6 Test accuracy: 0.5993  Test loss: 1.17538
7 154 Train accuracy: 0.605098050316  Train loss: 1.13510882611
7 Test accuracy: 0.5991  Test loss: 1.1479
8 154 Train accuracy: 0.621456345144  Train loss: 1.09074657738
8 Test accuracy: 0.6149  Test loss: 1.13297
9 154 Train accuracy: 0.636649971798  Train loss: 1.05320552133
9 Test accuracy: 0.5912  Test loss: 1.22807
10 154 Train accuracy: 0.649527496845  Train loss: 1.01529427565
10 Test accuracy: 0.5996  Test loss: 1.23189
11 154 Train accuracy: 0.66026578621  Train loss: 0.9850117316498
11 Test accuracy: 0.6406  Test loss: 1.06694
12 154 Train accuracy: 0.671020919989  Train loss: 0.955184110573
12 Test accuracy: 0.6309  Test loss: 1.10703
13 154 Train accuracy: 0.681104870206  Train loss: 0.927786078754
13 Test accuracy: 0.6274  Test loss: 1.13324
14 154 Train accuracy: 0.691812895314  Train loss: 0.898661519898
14 Test accuracy: 0.6207  Test loss: 1.16254
15 154 Train accuracy: 0.701500817637  Train loss: 0.871445595083
15 Test accuracy: 0.633  Test loss: 1.15964
16 154 Train accuracy: 0.710429389495  Train loss: 0.846427196903
Early stopping, best loss:  1.06694
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 32, 'activation': 'elu', 'ksize3': 5, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 128}
1 310 Train accuracy: 0.504472276339  Train loss: 1.42278778553
1 Test accuracy: 0.5289  Test loss: 1.32359
2 310 Train accuracy: 0.585569472267  Train loss: 1.18711508925
2 Test accuracy: 0.5715  Test loss: 1.24812
3 310 Train accuracy: 0.635460146727  Train loss: 1.04451148632
3 Test accuracy: 0.5434  Test loss: 1.40194
4 310 Train accuracy: 0.672778772047  Train loss: 0.942962459647
4 Test accuracy: 0.6784  Test loss: 0.92974
5 310 Train accuracy: 0.703518186624  Train loss: 0.860045585266
5 Test accuracy: 0.6743  Test loss: 0.95727
6 310 Train accuracy: 0.728781553033  Train loss: 0.786565366082
6 Test accuracy: 0.688  Test loss: 0.962286
7 310 Train accuracy: 0.751086122715  Train loss: 0.724094683652
7 Test accuracy: 0.6879  Test loss: 1.01069
8 310 Train accuracy: 0.771615980623  Train loss: 0.665191640504
8 Test accuracy: 0.6256  Test loss: 1.36613
9 310 Train accuracy: 0.78957132422  Train loss: 0.6161334761747
Early stopping, best loss:  0.92974
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 64, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 96, 'activation': 'lrelu', 'ksize3': 4, 'ksize2': 4, 'filters3': 128, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 64}
1 623 Train accuracy: 0.538461557925  Train loss: 1.30539076805
1 Test accuracy: 0.5272  Test loss: 1.34995
2 623 Train accuracy: 0.613846172839  Train loss: 1.10120519876
2 Test accuracy: 0.5917  Test loss: 1.14692
3 623 Train accuracy: 0.658871812522  Train loss: 0.988681130409
3 Test accuracy: 0.6323  Test loss: 1.05846
4 623 Train accuracy: 0.695384631529  Train loss: 0.890797536075
4 Test accuracy: 0.6524  Test loss: 0.981784
5 623 Train accuracy: 0.721600014985  Train loss: 0.819557599306
5 Test accuracy: 0.6647  Test loss: 0.969241
6 623 Train accuracy: 0.743487193435  Train loss: 0.761171343327
6 Test accuracy: 0.654  Test loss: 1.06365
7 600 Train accuracy: 0.762813199801  Train loss: 0.709018062694623 Train accuracy: 0.762813199801  Train loss: 0.709018062694
7 Test accuracy: 0.6573  Test loss: 1.02481
8 600 Train accuracy: 0.780000012107  Train loss: 0.664230809137623 Train accuracy: 0.780000012107  Train loss: 0.664230809137
8 Test accuracy: 0.6815  Test loss: 1.01364
9 623 Train accuracy: 0.795555566847  Train loss: 0.620915141569
9 Test accuracy: 0.6793  Test loss: 1.04231
10 623 Train accuracy: 0.810461548954  Train loss: 0.581664242744
Early stopping, best loss:  0.969241
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 80, 'optimizer': 'rmsprop', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 16, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 64}
1 1248 Train accuracy: 0.418787887692  Train loss: 1.58379388094
1 Test accuracy: 0.3983  Test loss: 1.74147
2 1248 Train accuracy: 0.462121219933  Train loss: 1.45682814598
2 Test accuracy: 0.5012  Test loss: 1.63845
3 1248 Train accuracy: 0.494545462529  Train loss: 1.38014821728
3 Test accuracy: 0.5276  Test loss: 1.39024
4 1248 Train accuracy: 0.525000007451  Train loss: 1.30433914908
4 Test accuracy: 0.515  Test loss: 1.34958
5 1248 Train accuracy: 0.545696976185  Train loss: 1.24902718163
5 Test accuracy: 0.5877  Test loss: 1.28155
6 1248 Train accuracy: 0.561515157719  Train loss: 1.21190368434
6 Test accuracy: 0.5397  Test loss: 1.40305
7 1248 Train accuracy: 0.577835503561  Train loss: 1.17154515249
7 Test accuracy: 0.5568  Test loss: 1.37582
8 1248 Train accuracy: 0.589772733077  Train loss: 1.14539270386
8 Test accuracy: 0.549  Test loss: 1.40249
9 1248 Train accuracy: 0.601750847565  Train loss: 1.11731760316
9 Test accuracy: 0.6122  Test loss: 1.3117
10 1225 Train accuracy: 0.611575763404  Train loss: 1.093650859381248 Train accuracy: 0.611575763404  Train loss: 1.0936508593
Early stopping, best loss:  1.28155
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 64, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 16, 'activation': 'relu', 'ksize3': 5, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 64}
1 623 Train accuracy: 0.51015386343  Train loss: 1.375687031754
1 Test accuracy: 0.5013  Test loss: 1.37353
2 623 Train accuracy: 0.574769249558  Train loss: 1.18166707516
2 Test accuracy: 0.6127  Test loss: 1.12368
3 623 Train accuracy: 0.619897454182  Train loss: 1.08241015752
3 Test accuracy: 0.6353  Test loss: 1.04215
4 623 Train accuracy: 0.651692325175  Train loss: 0.987272501886
4 Test accuracy: 0.6659  Test loss: 0.965862
5 600 Train accuracy: 0.677292324305  Train loss: 0.918151208401623 Train accuracy: 0.677292324305  Train loss: 0.918151208401
5 Test accuracy: 0.6549  Test loss: 1.05742
6 623 Train accuracy: 0.701743605336  Train loss: 0.850783611337
6 Test accuracy: 0.6726  Test loss: 1.00072
7 623 Train accuracy: 0.724659355198  Train loss: 0.792448123693
7 Test accuracy: 0.6923  Test loss: 0.946625
8 623 Train accuracy: 0.742538475245  Train loss: 0.741763111576
8 Test accuracy: 0.677  Test loss: 1.03277
9 623 Train accuracy: 0.759794884655  Train loss: 0.696132025917
9 Test accuracy: 0.6936  Test loss: 1.00827
10 623 Train accuracy: 0.774953858256  Train loss: 0.655091899216
10 Test accuracy: 0.6519  Test loss: 1.23389
11 623 Train accuracy: 0.788475535891  Train loss: 0.617139443864
11 Test accuracy: 0.6742  Test loss: 1.18692
12 623 Train accuracy: 0.801025651793  Train loss: 0.583116896575
Early stopping, best loss:  0.946625
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 32, 'learning_rate': 0.001, 'ksize1': 4, 'filters1': 32, 'activation': 'elu', 'ksize3': 3, 'ksize2': 4, 'filters3': 96, 'patch_reduction': 0, 'momentum': 0.99, 'filters2': 48}
1 1248 Train accuracy: 0.540000006557  Train loss: 1.26214837551
1 Test accuracy: 0.5454  Test loss: 1.32898
2 1248 Train accuracy: 0.605757583082  Train loss: 1.09656085134
2 Test accuracy: 0.6282  Test loss: 1.04586
3 1248 Train accuracy: 0.651515158216  Train loss: 0.984563917518
3 Test accuracy: 0.5775  Test loss: 1.2449
4 1248 Train accuracy: 0.676515158862  Train loss: 0.914486447722
4 Test accuracy: 0.6377  Test loss: 1.02768
5 1248 Train accuracy: 0.698303038478  Train loss: 0.855169631614
5 Test accuracy: 0.687  Test loss: 0.904974
6 1248 Train accuracy: 0.719191927612  Train loss: 0.796908726643
6 Test accuracy: 0.6895  Test loss: 0.94335
7 1248 Train accuracy: 0.738181826983  Train loss: 0.747421845244
7 Test accuracy: 0.6954  Test loss: 0.94487
8 1248 Train accuracy: 0.754545463696  Train loss: 0.705141133964
8 Test accuracy: 0.6694  Test loss: 1.08845
9 1248 Train accuracy: 0.76868687835  Train loss: 0.6649935043529
9 Test accuracy: 0.6942  Test loss: 1.05659
10 1248 Train accuracy: 0.781636373937  Train loss: 0.629661106199
Early stopping, best loss:  0.904974
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 256, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 32, 'activation': 'relu', 'ksize3': 3, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 64}
1 150 Train accuracy: 0.475819881473  Train loss: 1.46132319314154 Train accuracy: 0.475819881473  Train loss: 1.46132319314
1 Test accuracy: 0.5509  Test loss: 1.24606
2 154 Train accuracy: 0.548638111779  Train loss: 1.27864196897
2 Test accuracy: 0.5465  Test loss: 1.2915
3 154 Train accuracy: 0.599221770253  Train loss: 1.15073062976
3 Test accuracy: 0.6277  Test loss: 1.07788
4 154 Train accuracy: 0.625625328294  Train loss: 1.06756327621
4 Test accuracy: 0.5526  Test loss: 1.36418
5 154 Train accuracy: 0.652028886761  Train loss: 0.99395101411
5 Test accuracy: 0.6244  Test loss: 1.10418
6 154 Train accuracy: 0.673244377687  Train loss: 0.931912657769
6 Test accuracy: 0.6678  Test loss: 0.983491
7 154 Train accuracy: 0.690701166586  Train loss: 0.881620638225
7 Test accuracy: 0.6307  Test loss: 1.13356
8 154 Train accuracy: 0.707128944674  Train loss: 0.834799722369
8 Test accuracy: 0.6492  Test loss: 1.10347
9 154 Train accuracy: 0.723858918936  Train loss: 0.790170827555
9 Test accuracy: 0.6497  Test loss: 1.12248
10 154 Train accuracy: 0.736687033943  Train loss: 0.754302448034
10 Test accuracy: 0.6684  Test loss: 1.01378
11 154 Train accuracy: 0.750366351434  Train loss: 0.716269789578
Early stopping, best loss:  0.983491
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 32, 'learning_rate': 0.0015, 'ksize1': 4, 'filters1': 32, 'activation': 'lrelu', 'ksize3': 5, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 128}
1 1248 Train accuracy: 0.574545459151  Train loss: 1.23852447152
1 Test accuracy: 0.5965  Test loss: 1.12977
2 1248 Train accuracy: 0.633333338201  Train loss: 1.06234773636
2 Test accuracy: 0.637  Test loss: 1.05802
3 1248 Train accuracy: 0.67959596614  Train loss: 0.9398831687378
3 Test accuracy: 0.606  Test loss: 1.12695
4 1248 Train accuracy: 0.714545462877  Train loss: 0.845817063004
4 Test accuracy: 0.6462  Test loss: 1.00932
5 1248 Train accuracy: 0.746787887454  Train loss: 0.762321231127
5 Test accuracy: 0.6911  Test loss: 0.905265
6 1248 Train accuracy: 0.773636372983  Train loss: 0.692895140449
6 Test accuracy: 0.6839  Test loss: 0.941905
7 1248 Train accuracy: 0.796363646388  Train loss: 0.629178272997 Train accuracy: 0.796363646388  Train loss: 0.629178272997
7 Test accuracy: 0.6548  Test loss: 1.15063
8 1248 Train accuracy: 0.817500009909  Train loss: 0.572488023276
8 Test accuracy: 0.6918  Test loss: 1.04633
9 1248 Train accuracy: 0.835218865143  Train loss: 0.522939232331
9 Test accuracy: 0.7124  Test loss: 1.0049
10 1248 Train accuracy: 0.850787888348  Train loss: 0.478262825996
Early stopping, best loss:  0.905265
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 256, 'learning_rate': 0.002, 'ksize1': 5, 'filters1': 64, 'activation': 'lrelu', 'ksize3': 4, 'ksize2': 3, 'filters3': 128, 'patch_reduction': 0, 'momentum': 0.95, 'filters2': 128}
1 154 Train accuracy: 0.460811544742  Train loss: 1.70329282965
1 Test accuracy: 0.4523  Test loss: 1.54035
2 154 Train accuracy: 0.544191197625  Train loss: 1.36911032455
2 Test accuracy: 0.4963  Test loss: 1.56238
3 154 Train accuracy: 0.598110042158  Train loss: 1.17975824504
3 Test accuracy: 0.6503  Test loss: 0.999597
4 154 Train accuracy: 0.637020549072  Train loss: 1.05356389923
4 Test accuracy: 0.6771  Test loss: 0.941286
5 154 Train accuracy: 0.669038337895  Train loss: 0.952633844103
5 Test accuracy: 0.681  Test loss: 0.904818
6 154 Train accuracy: 0.698443564276  Train loss: 0.872307059311
6 Test accuracy: 0.6722  Test loss: 1.04083
7 154 Train accuracy: 0.723735394222  Train loss: 0.799119138596
7 Test accuracy: 0.6481  Test loss: 1.19172
8 154 Train accuracy: 0.745622554794  Train loss: 0.737449809377
8 Test accuracy: 0.6877  Test loss: 1.08053
9 154 Train accuracy: 0.766536952721  Train loss: 0.679878071424
9 Test accuracy: 0.683  Test loss: 1.25784
10 154 Train accuracy: 0.785436342231  Train loss: 0.627231142351
Early stopping, best loss:  0.904818
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 256, 'learning_rate': 0.002, 'ksize1': 3, 'filters1': 64, 'activation': 'relu', 'ksize3': 4, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.99, 'filters2': 128}
1 154 Train accuracy: 0.421345177506  Train loss: 1.57766560146
1 Test accuracy: 0.3684  Test loss: 1.80221
2 154 Train accuracy: 0.508337948471  Train loss: 1.34881397231
2 Test accuracy: 0.4866  Test loss: 1.64474
3 154 Train accuracy: 0.569946248971  Train loss: 1.18858586323
3 Test accuracy: 0.5712  Test loss: 1.25526
4 154 Train accuracy: 0.611033890662  Train loss: 1.0829403896
4 Test accuracy: 0.574  Test loss: 1.26913
5 154 Train accuracy: 0.642579194265  Train loss: 1.00139218739
5 Test accuracy: 0.5479  Test loss: 1.46286
6 154 Train accuracy: 0.666851939013  Train loss: 0.936379541953
6 Test accuracy: 0.6069  Test loss: 1.21618
7 154 Train accuracy: 0.686413071441  Train loss: 0.882104565294
7 Test accuracy: 0.5626  Test loss: 1.49467
8 154 Train accuracy: 0.70532238683  Train loss: 0.8316480022466
8 Test accuracy: 0.6671  Test loss: 1.1447
9 154 Train accuracy: 0.721635462628  Train loss: 0.786615673512
9 Test accuracy: 0.6802  Test loss: 1.08421
10 154 Train accuracy: 0.736297930032  Train loss: 0.746018225383
10 Test accuracy: 0.6749  Test loss: 1.17689
11 154 Train accuracy: 0.748951424929  Train loss: 0.711359963402
11 Test accuracy: 0.6478  Test loss: 1.31506
12 154 Train accuracy: 0.760931987049  Train loss: 0.677926159331
12 Test accuracy: 0.6565  Test loss: 1.24635
13 154 Train accuracy: 0.771240420967  Train loss: 0.649774199495
13 Test accuracy: 0.6661  Test loss: 1.408
14 150 Train accuracy: 0.780989427272  Train loss: 0.621891905459154 Train accuracy: 0.780989427272  Train loss: 0.621891905459
Early stopping, best loss:  1.08421
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 64, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 96, 'activation': 'elu', 'ksize3': 3, 'ksize2': 5, 'filters3': 96, 'patch_reduction': 0, 'momentum': 0.95, 'filters2': 128}
1 623 Train accuracy: 0.568000019193  Train loss: 1.20428236246
1 Test accuracy: 0.5118  Test loss: 1.3766
2 623 Train accuracy: 0.636615402997  Train loss: 1.03017341733
2 Test accuracy: 0.6127  Test loss: 1.10031
3 623 Train accuracy: 0.682461555203  Train loss: 0.924810563723
3 Test accuracy: 0.6566  Test loss: 1.00051
4 623 Train accuracy: 0.713538476974  Train loss: 0.844318150282
4 Test accuracy: 0.6494  Test loss: 1.03049
5 623 Train accuracy: 0.744615398526  Train loss: 0.767849264622
5 Test accuracy: 0.6494  Test loss: 1.05653
6 600 Train accuracy: 0.768307705025  Train loss: 0.702332572738623 Train accuracy: 0.768307705025  Train loss: 0.702332572738
6 Test accuracy: 0.6458  Test loss: 1.10994
7 623 Train accuracy: 0.789098912733  Train loss: 0.645441901599
7 Test accuracy: 0.6596  Test loss: 1.11517
8 623 Train accuracy: 0.808692318276  Train loss: 0.592684619948
Early stopping, best loss:  1.00051
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 96, 'activation': 'elu', 'ksize3': 3, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.9, 'filters2': 64}
1 310 Train accuracy: 0.446034585054  Train loss: 1.58776508845
1 Test accuracy: 0.4868  Test loss: 1.46926
2 310 Train accuracy: 0.518187237474  Train loss: 1.39038439439
2 Test accuracy: 0.4608  Test loss: 1.51087
3 310 Train accuracy: 0.556748160185  Train loss: 1.27191836559
3 Test accuracy: 0.5533  Test loss: 1.25288
4 310 Train accuracy: 0.584078712532  Train loss: 1.19532620104
4 Test accuracy: 0.6056  Test loss: 1.12591
5 310 Train accuracy: 0.606440072335  Train loss: 1.13464956115
5 Test accuracy: 0.5485  Test loss: 1.35081
6 300 Train accuracy: 0.6265156036  Train loss: 1.0792094148149310 Train accuracy: 0.6265156036  Train loss: 1.07920941481
6 Test accuracy: 0.6356  Test loss: 1.03298
7 310 Train accuracy: 0.644944203096  Train loss: 1.02890685561
7 Test accuracy: 0.5984  Test loss: 1.17987
8 300 Train accuracy: 0.658914729093  Train loss: 0.992685596532310 Train accuracy: 0.658914729093  Train loss: 0.99268559653
8 Test accuracy: 0.6004  Test loss: 1.1923
9 310 Train accuracy: 0.669316901865  Train loss: 0.958666616525
9 Test accuracy: 0.6417  Test loss: 1.0551
10 310 Train accuracy: 0.68109719822  Train loss: 0.9246753545912
10 Test accuracy: 0.6311  Test loss: 1.07539
11 310 Train accuracy: 0.692741367575  Train loss: 0.892430401557
Early stopping, best loss:  1.03298
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 96, 'activation': 'relu', 'ksize3': 5, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 96}
1 310 Train accuracy: 0.472868216725  Train loss: 1.48329103918
1 Test accuracy: 0.3747  Test loss: 1.99804
2 310 Train accuracy: 0.55873583773  Train loss: 1.235908641262
2 Test accuracy: 0.6112  Test loss: 1.10549
3 310 Train accuracy: 0.605843768288  Train loss: 1.10355450712
3 Test accuracy: 0.5941  Test loss: 1.1547
4 310 Train accuracy: 0.645348837456  Train loss: 1.00193843819
4 Test accuracy: 0.6226  Test loss: 1.16021
5 310 Train accuracy: 0.682289801882  Train loss: 0.906855512124
5 Test accuracy: 0.6503  Test loss: 1.04128
6 300 Train accuracy: 0.709004171957  Train loss: 0.835495656117310 Train accuracy: 0.709004171957  Train loss: 0.835495656117
6 Test accuracy: 0.6386  Test loss: 1.10141
7 310 Train accuracy: 0.733537779241  Train loss: 0.768633756664
7 Test accuracy: 0.6675  Test loss: 1.05596
8 310 Train accuracy: 0.755813953252  Train loss: 0.708703394549
8 Test accuracy: 0.7008  Test loss: 1.0327
9 310 Train accuracy: 0.775127541688  Train loss: 0.654851193866
9 Test accuracy: 0.6054  Test loss: 1.60999
10 310 Train accuracy: 0.792248060497  Train loss: 0.608028285721
10 Test accuracy: 0.6955  Test loss: 1.0976
11 310 Train accuracy: 0.808640968862  Train loss: 0.563860667231
11 Test accuracy: 0.7165  Test loss: 1.08192
12 310 Train accuracy: 0.82264956937  Train loss: 0.5249434998021
12 Test accuracy: 0.6967  Test loss: 1.3438
13 310 Train accuracy: 0.834457131092  Train loss: 0.491640719489
Early stopping, best loss:  1.0327
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 64, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 16, 'activation': 'relu', 'ksize3': 4, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 2, 'momentum': 0.99, 'filters2': 96}
1 623 Train accuracy: 0.496615402102  Train loss: 1.38333237648
1 Test accuracy: 0.4222  Test loss: 1.76475
2 623 Train accuracy: 0.564307712018  Train loss: 1.20396579623
2 Test accuracy: 0.5096  Test loss: 1.46967
3 623 Train accuracy: 0.603487198949  Train loss: 1.10373029232
3 Test accuracy: 0.5805  Test loss: 1.21674
4 623 Train accuracy: 0.638153864592  Train loss: 1.01221306175
4 Test accuracy: 0.6469  Test loss: 1.01401
5 623 Train accuracy: 0.663630786777  Train loss: 0.940840923786
5 Test accuracy: 0.6698  Test loss: 0.98989
6 623 Train accuracy: 0.689128221571  Train loss: 0.875038314064
6 Test accuracy: 0.6393  Test loss: 1.08742
7 623 Train accuracy: 0.710505509973  Train loss: 0.818718389613
7 Test accuracy: 0.6619  Test loss: 1.0352
8 623 Train accuracy: 0.728923091516  Train loss: 0.768069429845
8 Test accuracy: 0.6608  Test loss: 1.11717
9 623 Train accuracy: 0.743589757482  Train loss: 0.725479792224
9 Test accuracy: 0.6885  Test loss: 1.0421
10 623 Train accuracy: 0.75895385927  Train loss: 0.6853661756525
Early stopping, best loss:  0.98989
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 32, 'activation': 'elu', 'ksize3': 3, 'ksize2': 4, 'filters3': 96, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 64}
1 1248 Train accuracy: 0.53939394474  Train loss: 1.279919507588
1 Test accuracy: 0.4086  Test loss: 1.74589
2 1248 Train accuracy: 0.59696970284  Train loss: 1.129296075111
2 Test accuracy: 0.5907  Test loss: 1.17548
3 1248 Train accuracy: 0.631919197639  Train loss: 1.03637271603
3 Test accuracy: 0.5971  Test loss: 1.13981
4 1248 Train accuracy: 0.657575763762  Train loss: 0.971115839332
4 Test accuracy: 0.6607  Test loss: 0.972739
5 1248 Train accuracy: 0.682666672945  Train loss: 0.900012372971
5 Test accuracy: 0.6806  Test loss: 0.928317
6 1248 Train accuracy: 0.7060606124  Train loss: 0.84111397286396
6 Test accuracy: 0.6605  Test loss: 0.997129
7 1248 Train accuracy: 0.724069271088  Train loss: 0.795271123222
7 Test accuracy: 0.6507  Test loss: 1.05569
8 1248 Train accuracy: 0.74272728011  Train loss: 0.7462105465311
8 Test accuracy: 0.6881  Test loss: 0.941643
9 1248 Train accuracy: 0.757710445325  Train loss: 0.705162839724
9 Test accuracy: 0.6237  Test loss: 1.249
10 1248 Train accuracy: 0.771636371613  Train loss: 0.667004712939
Early stopping, best loss:  0.928317
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 125, 'optimizer': 'rmsprop', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 96, 'activation': 'elu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.9, 'filters2': 48}
1 310 Train accuracy: 0.42814550033  Train loss: 1.578523461644
1 Test accuracy: 0.3734  Test loss: 2.32482
2 310 Train accuracy: 0.51222420885  Train loss: 1.391207722528
2 Test accuracy: 0.3227  Test loss: 2.83668
3 310 Train accuracy: 0.553567878711  Train loss: 1.27481491749
3 Test accuracy: 0.4924  Test loss: 1.62119
4 310 Train accuracy: 0.586016696233  Train loss: 1.18579266392
4 Test accuracy: 0.6416  Test loss: 1.0699
5 310 Train accuracy: 0.610017888363  Train loss: 1.11926946915
5 Test accuracy: 0.4522  Test loss: 1.8228
6 310 Train accuracy: 0.628702045251  Train loss: 1.06499826144
6 Test accuracy: 0.522  Test loss: 1.82148
7 310 Train accuracy: 0.649970184316  Train loss: 1.00761695157
7 Test accuracy: 0.6214  Test loss: 1.18514
8 310 Train accuracy: 0.666964816359  Train loss: 0.958614924206
8 Test accuracy: 0.6149  Test loss: 1.30654
9 310 Train accuracy: 0.682236796261  Train loss: 0.920244232202
Early stopping, best loss:  1.0699
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 100, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 4, 'filters1': 16, 'activation': 'elu', 'ksize3': 3, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 2, 'momentum': 0.99, 'filters2': 128}
1 310 Train accuracy: 0.45378652215  Train loss: 1.526276661888
1 Test accuracy: 0.4706  Test loss: 1.4706
2 310 Train accuracy: 0.531305903426  Train loss: 1.33710433428
2 Test accuracy: 0.5576  Test loss: 1.25007
3 310 Train accuracy: 0.577618767818  Train loss: 1.21195116257
3 Test accuracy: 0.5663  Test loss: 1.24767
4 310 Train accuracy: 0.608974362222  Train loss: 1.13103864743
4 Test accuracy: 0.571  Test loss: 1.24677
5 310 Train accuracy: 0.636135961001  Train loss: 1.05833593424
5 Test accuracy: 0.6224  Test loss: 1.10055
6 310 Train accuracy: 0.657821510465  Train loss: 0.997066827921
6 Test accuracy: 0.5946  Test loss: 1.25211
7 310 Train accuracy: 0.675611213996  Train loss: 0.951205098367
7 Test accuracy: 0.6292  Test loss: 1.10795
8 310 Train accuracy: 0.694469292863  Train loss: 0.900044916914
8 Test accuracy: 0.5852  Test loss: 1.26189
9 310 Train accuracy: 0.710528060157  Train loss: 0.855975301347
9 Test accuracy: 0.5968  Test loss: 1.31945
10 310 Train accuracy: 0.725223613932  Train loss: 0.816901860558
Early stopping, best loss:  1.10055
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 64, 'activation': 'relu', 'ksize3': 4, 'ksize2': 4, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.9, 'filters2': 96}
1 1248 Train accuracy: 0.376363645792  Train loss: 1.64023087502
1 Test accuracy: 0.2951  Test loss: 2.28515
2 1248 Train accuracy: 0.436060613394  Train loss: 1.50446855903
2 Test accuracy: 0.4731  Test loss: 1.51629
3 1248 Train accuracy: 0.472525259256  Train loss: 1.43276330034
3 Test accuracy: 0.4899  Test loss: 1.42783
4 1248 Train accuracy: 0.500606067479  Train loss: 1.36730821967
4 Test accuracy: 0.5446  Test loss: 1.26033
5 1225 Train accuracy: 0.519515157819  Train loss: 1.321141293051248 Train accuracy: 0.519515157819  Train loss: 1.32114129305
5 Test accuracy: 0.5217  Test loss: 1.4198
6 1248 Train accuracy: 0.532323238154  Train loss: 1.29454473277
6 Test accuracy: 0.5801  Test loss: 1.213
7 1248 Train accuracy: 0.546320351788  Train loss: 1.25883500612
7 Test accuracy: 0.5828  Test loss: 1.18232
8 1248 Train accuracy: 0.557121217623  Train loss: 1.22700220704
8 Test accuracy: 0.5639  Test loss: 1.33804
9 1248 Train accuracy: 0.567272732986  Train loss: 1.20499886844
9 Test accuracy: 0.6113  Test loss: 1.24569
10 1248 Train accuracy: 0.576060611606  Train loss: 1.18309668124
10 Test accuracy: 0.5618  Test loss: 1.29491
11 1248 Train accuracy: 0.585344358141  Train loss: 1.15656477554
11 Test accuracy: 0.6188  Test loss: 1.24834
12 1248 Train accuracy: 0.59545455118  Train loss: 1.129486260665
Early stopping, best loss:  1.18232
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 32, 'activation': 'elu', 'ksize3': 5, 'ksize2': 4, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.99, 'filters2': 128}
1 1248 Train accuracy: 0.352727280855  Train loss: 1.79515711784
1 Test accuracy: 0.2826  Test loss: 2.38103
2 1248 Train accuracy: 0.403636371493  Train loss: 1.69776442528
2 Test accuracy: 0.4539  Test loss: 1.60649
3 1248 Train accuracy: 0.455959602594  Train loss: 1.58089886189
3 Test accuracy: 0.4633  Test loss: 1.6117
4 1248 Train accuracy: 0.479242430925  Train loss: 1.51929367036
4 Test accuracy: 0.4889  Test loss: 1.53031
5 1248 Train accuracy: 0.502303035736  Train loss: 1.45454380345
5 Test accuracy: 0.4495  Test loss: 1.99021
6 1248 Train accuracy: 0.518989904225  Train loss: 1.41430423896
6 Test accuracy: 0.4889  Test loss: 2.32571
7 1248 Train accuracy: 0.540173165372  Train loss: 1.36598017216
7 Test accuracy: 0.5613  Test loss: 1.56485
8 1248 Train accuracy: 0.556515156254  Train loss: 1.32480484024
8 Test accuracy: 0.5485  Test loss: 1.66653
9 1248 Train accuracy: 0.567946132885  Train loss: 1.29675678147
Early stopping, best loss:  1.53031
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'rmsprop', 'batch_size': 128, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 64, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 64}
1 310 Train accuracy: 0.432319622773  Train loss: 1.51343609737
1 Test accuracy: 0.4398  Test loss: 1.79766
2 310 Train accuracy: 0.516100179691  Train loss: 1.32649480382
2 Test accuracy: 0.469  Test loss: 1.80608
3 310 Train accuracy: 0.563705028632  Train loss: 1.21593194589
3 Test accuracy: 0.5716  Test loss: 1.27328
4 310 Train accuracy: 0.598539056686  Train loss: 1.12799684359
4 Test accuracy: 0.5272  Test loss: 1.58507
5 310 Train accuracy: 0.621228382221  Train loss: 1.06327122454
5 Test accuracy: 0.5724  Test loss: 1.46043
6 310 Train accuracy: 0.642516396749  Train loss: 1.00821407789
6 Test accuracy: 0.6733  Test loss: 0.974435
7 310 Train accuracy: 0.659085100169  Train loss: 0.963810589287
7 Test accuracy: 0.6142  Test loss: 1.18441
8 310 Train accuracy: 0.673673226283  Train loss: 0.924209411328
8 Test accuracy: 0.6659  Test loss: 1.02811
9 310 Train accuracy: 0.687073478841  Train loss: 0.891681455139
9 Test accuracy: 0.6615  Test loss: 1.04932
10 310 Train accuracy: 0.698032201712  Train loss: 0.861617199733
10 Test accuracy: 0.6631  Test loss: 1.09052
11 310 Train accuracy: 0.708787337883  Train loss: 0.833382993728
Early stopping, best loss:  0.974435
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 32, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 64}
1 1248 Train accuracy: 0.401818191409  Train loss: 1.62784154177
1 Test accuracy: 0.3537  Test loss: 1.9046
2 1248 Train accuracy: 0.463333340585  Train loss: 1.46836466014
2 Test accuracy: 0.4558  Test loss: 1.90098
3 1248 Train accuracy: 0.502626268665  Train loss: 1.37049589674
3 Test accuracy: 0.4899  Test loss: 1.57973
4 1248 Train accuracy: 0.529545459896  Train loss: 1.30239706218
4 Test accuracy: 0.4521  Test loss: 1.71166
5 1248 Train accuracy: 0.550424247622  Train loss: 1.25350802088
5 Test accuracy: 0.5499  Test loss: 1.3899
6 1248 Train accuracy: 0.569090914031  Train loss: 1.20776445627
6 Test accuracy: 0.5517  Test loss: 1.48282
7 1248 Train accuracy: 0.580519485303  Train loss: 1.17229142632
7 Test accuracy: 0.5721  Test loss: 1.28979
8 1248 Train accuracy: 0.595151519626  Train loss: 1.13933812968
8 Test accuracy: 0.5386  Test loss: 1.73475
9 1248 Train accuracy: 0.608619533512  Train loss: 1.11022534894
9 Test accuracy: 0.6164  Test loss: 1.18863
10 1248 Train accuracy: 0.620303034902  Train loss: 1.07808946651
10 Test accuracy: 0.6147  Test loss: 1.23962
11 1248 Train accuracy: 0.631460060206  Train loss: 1.04862644254
11 Test accuracy: 0.6215  Test loss: 1.21459
12 1248 Train accuracy: 0.640252530575  Train loss: 1.02504638294
12 Test accuracy: 0.5806  Test loss: 1.28195
13 1248 Train accuracy: 0.647179492529  Train loss: 1.00505400415
13 Test accuracy: 0.5968  Test loss: 1.37027
14 1248 Train accuracy: 0.656017321689  Train loss: 0.981986127027
Early stopping, best loss:  1.18863
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'rmsprop', 'batch_size': 64, 'learning_rate': 0.0015, 'ksize1': 4, 'filters1': 16, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 3, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 48}
1 623 Train accuracy: 0.481846169233  Train loss: 1.49087086678
1 Test accuracy: 0.5249  Test loss: 1.36692
2 623 Train accuracy: 0.539076941609  Train loss: 1.30871801257
2 Test accuracy: 0.536  Test loss: 1.31992
3 623 Train accuracy: 0.574974378347  Train loss: 1.20770038366
3 Test accuracy: 0.6016  Test loss: 1.1703
4 623 Train accuracy: 0.601846173108  Train loss: 1.12613279462
4 Test accuracy: 0.5874  Test loss: 1.22369
5 623 Train accuracy: 0.62067694211  Train loss: 1.078392278672
5 Test accuracy: 0.5843  Test loss: 1.29938
6 623 Train accuracy: 0.638564121127  Train loss: 1.03088070552
6 Test accuracy: 0.5995  Test loss: 1.25146
7 623 Train accuracy: 0.649230787584  Train loss: 0.997922271664
7 Test accuracy: 0.6515  Test loss: 1.08023
8 623 Train accuracy: 0.66030771032  Train loss: 0.9649908101568
8 Test accuracy: 0.6393  Test loss: 1.11112
9 623 Train accuracy: 0.671726513306  Train loss: 0.932609922886
9 Test accuracy: 0.6559  Test loss: 1.08689
10 623 Train accuracy: 0.682030786395  Train loss: 0.902784198642
10 Test accuracy: 0.5887  Test loss: 1.55207
11 623 Train accuracy: 0.692307709022  Train loss: 0.876285267852
11 Test accuracy: 0.6452  Test loss: 1.18054
12 623 Train accuracy: 0.701641041934  Train loss: 0.851285691758
Early stopping, best loss:  1.08023
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 64, 'learning_rate': 0.001, 'ksize1': 4, 'filters1': 32, 'activation': 'relu', 'ksize3': 4, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 128}
1 600 Train accuracy: 0.455384627134  Train loss: 1.51000268936623 Train accuracy: 0.455384627134  Train loss: 1.51000268936
1 Test accuracy: 0.4687  Test loss: 1.55127
2 623 Train accuracy: 0.541230786219  Train loss: 1.29442915201
2 Test accuracy: 0.5642  Test loss: 1.29055
3 623 Train accuracy: 0.589333350907  Train loss: 1.16249586662
3 Test accuracy: 0.554  Test loss: 1.48037
4 623 Train accuracy: 0.62061540205  Train loss: 1.082553994062
4 Test accuracy: 0.5923  Test loss: 1.32471
5 623 Train accuracy: 0.641969247967  Train loss: 1.01455058956
5 Test accuracy: 0.6365  Test loss: 1.07645
6 623 Train accuracy: 0.659384632235  Train loss: 0.964922617873
6 Test accuracy: 0.5369  Test loss: 1.4842
7 623 Train accuracy: 0.676395620725  Train loss: 0.914442246131
7 Test accuracy: 0.6032  Test loss: 1.34991
8 623 Train accuracy: 0.689846169744  Train loss: 0.880468336046
8 Test accuracy: 0.6787  Test loss: 1.07323
9 623 Train accuracy: 0.701538477027  Train loss: 0.851811218659
9 Test accuracy: 0.6358  Test loss: 1.504
10 623 Train accuracy: 0.713476938084  Train loss: 0.821458376884
10 Test accuracy: 0.6095  Test loss: 1.51093
11 623 Train accuracy: 0.724923091436  Train loss: 0.789427004511
11 Test accuracy: 0.6795  Test loss: 1.2189
12 623 Train accuracy: 0.734153860273  Train loss: 0.764811997612
12 Test accuracy: 0.6606  Test loss: 1.32282
13 623 Train accuracy: 0.742816581783  Train loss: 0.740934428894
Early stopping, best loss:  1.07323
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 5, 'filters1': 32, 'activation': 'lrelu', 'ksize3': 5, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.99, 'filters2': 64}
1 310 Train accuracy: 0.485390574886  Train loss: 1.42064629151
1 Test accuracy: 0.5853  Test loss: 1.16785
2 310 Train accuracy: 0.583482407033  Train loss: 1.16438927559
2 Test accuracy: 0.5946  Test loss: 1.18518
3 310 Train accuracy: 0.635857677995  Train loss: 1.03219734421
3 Test accuracy: 0.6044  Test loss: 1.19849
4 310 Train accuracy: 0.675462133323  Train loss: 0.932190687037
4 Test accuracy: 0.6233  Test loss: 1.11008
5 310 Train accuracy: 0.701132974946  Train loss: 0.862850089257
5 Test accuracy: 0.5145  Test loss: 1.68107
6 310 Train accuracy: 0.724706816941  Train loss: 0.801864023013
6 Test accuracy: 0.6599  Test loss: 1.06078
7 310 Train accuracy: 0.741204530164  Train loss: 0.751128933259
7 Test accuracy: 0.6767  Test loss: 0.974018
8 310 Train accuracy: 0.758348239586  Train loss: 0.701144684249
8 Test accuracy: 0.7109  Test loss: 0.906563
9 300 Train accuracy: 0.774928775481  Train loss: 0.655732058052310 Train accuracy: 0.774928775481  Train loss: 0.655732058052
9 Test accuracy: 0.7113  Test loss: 0.909551
10 310 Train accuracy: 0.788849135431  Train loss: 0.616090077047
10 Test accuracy: 0.6714  Test loss: 1.14378
11 310 Train accuracy: 0.802298476959  Train loss: 0.579516532955
11 Test accuracy: 0.7045  Test loss: 1.08799
12 310 Train accuracy: 0.814698866067  Train loss: 0.544355589992
12 Test accuracy: 0.6768  Test loss: 1.37777
13 310 Train accuracy: 0.8261088922  Train loss: 0.51321451339576 Train accuracy: 0.8261088922  Train loss: 0.513214513395
Early stopping, best loss:  0.906563
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 64, 'learning_rate': 0.002, 'ksize1': 3, 'filters1': 96, 'activation': 'relu', 'ksize3': 3, 'ksize2': 5, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 96}
1 623 Train accuracy: 0.561230788231  Train loss: 1.22275492439
1 Test accuracy: 0.5747  Test loss: 1.17048
2 623 Train accuracy: 0.620923095345  Train loss: 1.07290492773
2 Test accuracy: 0.5782  Test loss: 1.18169
3 623 Train accuracy: 0.661538478931  Train loss: 0.966457163493
3 Test accuracy: 0.6714  Test loss: 0.924246
4 623 Train accuracy: 0.687692324221  Train loss: 0.891131476763
4 Test accuracy: 0.6358  Test loss: 1.09331
5 600 Train accuracy: 0.715200015306  Train loss: 0.820652891636623 Train accuracy: 0.715200015306  Train loss: 0.820652891636
5 Test accuracy: 0.6499  Test loss: 1.05063
6 623 Train accuracy: 0.736102578441  Train loss: 0.763113421202
6 Test accuracy: 0.6891  Test loss: 0.919391
7 623 Train accuracy: 0.75463737607  Train loss: 0.7111965801899
7 Test accuracy: 0.7117  Test loss: 0.919377
8 623 Train accuracy: 0.770153858811  Train loss: 0.666472062245
8 Test accuracy: 0.6904  Test loss: 1.00167
9 623 Train accuracy: 0.785367533233  Train loss: 0.623975360791
9 Test accuracy: 0.6917  Test loss: 1.06702
10 600 Train accuracy: 0.799200011134  Train loss: 0.585447888106623 Train accuracy: 0.799200011134  Train loss: 0.585447888106
10 Test accuracy: 0.7242  Test loss: 1.01286
11 600 Train accuracy: 0.81174826221  Train loss: 0.5500566525355623 Train accuracy: 0.81174826221  Train loss: 0.550056652535
11 Test accuracy: 0.6996  Test loss: 1.1219
12 600 Train accuracy: 0.822769240638  Train loss: 0.519190080538623 Train accuracy: 0.822769240638  Train loss: 0.519190080538
Early stopping, best loss:  0.919377
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 5, 'filters1': 96, 'activation': 'relu', 'ksize3': 4, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 128}
1 310 Train accuracy: 0.531902207778  Train loss: 1.27801853877
1 Test accuracy: 0.5574  Test loss: 1.25551
2 310 Train accuracy: 0.605843770962  Train loss: 1.08280407695
2 Test accuracy: 0.592  Test loss: 1.18728
3 310 Train accuracy: 0.65533691339  Train loss: 0.9617212124366
3 Test accuracy: 0.6049  Test loss: 1.29156
4 310 Train accuracy: 0.690966013532  Train loss: 0.868308431827
4 Test accuracy: 0.5934  Test loss: 1.33283
5 310 Train accuracy: 0.718783544577  Train loss: 0.800610554218
5 Test accuracy: 0.6777  Test loss: 0.984135
6 310 Train accuracy: 0.74349036125  Train loss: 0.7346128744968
6 Test accuracy: 0.7093  Test loss: 0.897427
7 310 Train accuracy: 0.764545534338  Train loss: 0.675690669935
7 Test accuracy: 0.6991  Test loss: 0.945366
8 310 Train accuracy: 0.782796663734  Train loss: 0.625650089664
8 Test accuracy: 0.7192  Test loss: 0.943922
9 310 Train accuracy: 0.800437290954  Train loss: 0.577663557142
9 Test accuracy: 0.698  Test loss: 1.04799
10 300 Train accuracy: 0.815980919049  Train loss: 0.536945551634310 Train accuracy: 0.815980919049  Train loss: 0.536945551634
10 Test accuracy: 0.7106  Test loss: 1.09293
11 310 Train accuracy: 0.829619991613  Train loss: 0.498899861649
Early stopping, best loss:  0.897427
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'rmsprop', 'batch_size': 64, 'learning_rate': 0.001, 'ksize1': 5, 'filters1': 96, 'activation': 'elu', 'ksize3': 3, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 48}
1 623 Train accuracy: 0.491076936126  Train loss: 1.43145190716
1 Test accuracy: 0.4379  Test loss: 1.72949
2 623 Train accuracy: 0.560615402162  Train loss: 1.22890973806
2 Test accuracy: 0.6357  Test loss: 1.05872
3 623 Train accuracy: 0.609641043146  Train loss: 1.11747116725
3 Test accuracy: 0.4941  Test loss: 1.67785
4 600 Train accuracy: 0.641538478583  Train loss: 1.03098601937623 Train accuracy: 0.641538478583  Train loss: 1.03098601937
4 Test accuracy: 0.6357  Test loss: 1.06049
5 623 Train accuracy: 0.666092324138  Train loss: 0.961927466631
5 Test accuracy: 0.665  Test loss: 0.990182
6 623 Train accuracy: 0.685128221015  Train loss: 0.907260318597
6 Test accuracy: 0.665  Test loss: 1.01659
7 623 Train accuracy: 0.699428586875  Train loss: 0.867088773421
7 Test accuracy: 0.6515  Test loss: 1.18682
8 623 Train accuracy: 0.71446155332  Train loss: 0.8242454242718
8 Test accuracy: 0.6494  Test loss: 1.1073
9 623 Train accuracy: 0.725811980234  Train loss: 0.789435099231
9 Test accuracy: 0.6845  Test loss: 0.991304
10 623 Train accuracy: 0.737415398538  Train loss: 0.757709525466
Early stopping, best loss:  0.990182
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 256, 'learning_rate': 0.0015, 'ksize1': 5, 'filters1': 96, 'activation': 'elu', 'ksize3': 5, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 128}
1 154 Train accuracy: 0.395219559116  Train loss: 1.71113559178
1 Test accuracy: 0.4838  Test loss: 1.45161
2 154 Train accuracy: 0.474986088595  Train loss: 1.48397910595
2 Test accuracy: 0.5469  Test loss: 1.26709
3 154 Train accuracy: 0.521400760682  Train loss: 1.36222945509
3 Test accuracy: 0.5191  Test loss: 1.33489
4 154 Train accuracy: 0.560033333887  Train loss: 1.25973479663
4 Test accuracy: 0.5804  Test loss: 1.17501
5 150 Train accuracy: 0.587437447267  Train loss: 1.18635565383154 Train accuracy: 0.587437447267  Train loss: 1.18635565383
5 Test accuracy: 0.586  Test loss: 1.17085
6 154 Train accuracy: 0.60718916126  Train loss: 1.125839562644
6 Test accuracy: 0.6284  Test loss: 1.06831
7 154 Train accuracy: 0.625982671216  Train loss: 1.07231626097
7 Test accuracy: 0.6213  Test loss: 1.08862
8 150 Train accuracy: 0.643482473147  Train loss: 1.02442564177154 Train accuracy: 0.643482473147  Train loss: 1.02442564177
8 Test accuracy: 0.6203  Test loss: 1.10533
9 154 Train accuracy: 0.658575735277  Train loss: 0.984427038639
9 Test accuracy: 0.6372  Test loss: 1.0727
10 154 Train accuracy: 0.672151178867  Train loss: 0.947677224874
10 Test accuracy: 0.6406  Test loss: 1.0674
11 154 Train accuracy: 0.685734483232  Train loss: 0.912477992959
11 Test accuracy: 0.6608  Test loss: 1.00313
12 150 Train accuracy: 0.69881414081  Train loss: 0.8779574238839154 Train accuracy: 0.69881414081  Train loss: 0.877957423883
12 Test accuracy: 0.6555  Test loss: 1.03744
13 154 Train accuracy: 0.711933965785  Train loss: 0.843500790366
13 Test accuracy: 0.6306  Test loss: 1.19264
14 154 Train accuracy: 0.72353687145  Train loss: 0.8137185725632
14 Test accuracy: 0.6509  Test loss: 1.07775
15 154 Train accuracy: 0.734704451831  Train loss: 0.784776530379
15 Test accuracy: 0.6302  Test loss: 1.17682
16 154 Train accuracy: 0.745205656692  Train loss: 0.756734200886
Early stopping, best loss:  1.00313
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 100, 'optimizer': 'nesterov', 'batch_size': 256, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 16, 'activation': 'relu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 48}
1 154 Train accuracy: 0.375208444893  Train loss: 1.80417663717
1 Test accuracy: 0.4737  Test loss: 1.48459
2 154 Train accuracy: 0.445247349462  Train loss: 1.59242927177
2 Test accuracy: 0.4757  Test loss: 1.45134
3 154 Train accuracy: 0.49231052789  Train loss: 1.457950160622
3 Test accuracy: 0.5262  Test loss: 1.3486
4 154 Train accuracy: 0.525569744142  Train loss: 1.35910123374
4 Test accuracy: 0.524  Test loss: 1.35931
5 154 Train accuracy: 0.547971077263  Train loss: 1.28893052685
5 Test accuracy: 0.5505  Test loss: 1.29108
6 154 Train accuracy: 0.568741875568  Train loss: 1.23162042385
6 Test accuracy: 0.6046  Test loss: 1.12029
7 154 Train accuracy: 0.589692668495  Train loss: 1.17354583862
7 Test accuracy: 0.6258  Test loss: 1.06104
8 154 Train accuracy: 0.605892144543  Train loss: 1.12546683741
8 Test accuracy: 0.6006  Test loss: 1.15131
9 154 Train accuracy: 0.617997635333  Train loss: 1.09194719413
9 Test accuracy: 0.6099  Test loss: 1.11905
10 154 Train accuracy: 0.632795980466  Train loss: 1.05508807387
10 Test accuracy: 0.5873  Test loss: 1.22101
11 150 Train accuracy: 0.645409048668  Train loss: 1.02164083642154 Train accuracy: 0.645409048668  Train loss: 1.02164083642
11 Test accuracy: 0.6149  Test loss: 1.12217
12 154 Train accuracy: 0.655086142233  Train loss: 0.992515192145
Early stopping, best loss:  1.06104
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 3, 'filters1': 32, 'activation': 'elu', 'ksize3': 5, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.95, 'filters2': 64}
1 1248 Train accuracy: 0.547878792882  Train loss: 1.27144696712
1 Test accuracy: 0.5319  Test loss: 1.33344
2 1248 Train accuracy: 0.610303033888  Train loss: 1.10425103784
2 Test accuracy: 0.5345  Test loss: 1.32865
3 1248 Train accuracy: 0.64808081309  Train loss: 1.001320553421
3 Test accuracy: 0.5952  Test loss: 1.14685
4 1248 Train accuracy: 0.677272732258  Train loss: 0.926588916638
4 Test accuracy: 0.6492  Test loss: 1.01002
5 1248 Train accuracy: 0.702545460224  Train loss: 0.859103091478
5 Test accuracy: 0.6618  Test loss: 0.971351
6 1248 Train accuracy: 0.723434349895  Train loss: 0.806493977904
6 Test accuracy: 0.6891  Test loss: 0.934515
7 1248 Train accuracy: 0.73913420592  Train loss: 0.7645509745821
7 Test accuracy: 0.6856  Test loss: 0.956002
8 1248 Train accuracy: 0.755909098387  Train loss: 0.719243123233
8 Test accuracy: 0.6791  Test loss: 0.968348
9 1248 Train accuracy: 0.770505058501  Train loss: 0.681441296676
9 Test accuracy: 0.6709  Test loss: 1.05504
10 1248 Train accuracy: 0.782121220708  Train loss: 0.648374207437
10 Test accuracy: 0.693  Test loss: 0.97985
11 1248 Train accuracy: 0.794325077425  Train loss: 0.615808613287
Early stopping, best loss:  0.934515
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 32, 'activation': 'relu', 'ksize3': 3, 'ksize2': 5, 'filters3': 96, 'patch_reduction': 1, 'momentum': 0.9, 'filters2': 48}
1 310 Train accuracy: 0.437686352203  Train loss: 1.61926058623
1 Test accuracy: 0.4536  Test loss: 1.48155
2 310 Train accuracy: 0.49522958266  Train loss: 1.429565995931
2 Test accuracy: 0.5278  Test loss: 1.32864
3 310 Train accuracy: 0.534287423086  Train loss: 1.32071131315
3 Test accuracy: 0.5595  Test loss: 1.25465
4 310 Train accuracy: 0.561568280395  Train loss: 1.23579239043
4 Test accuracy: 0.5732  Test loss: 1.22138
5 310 Train accuracy: 0.588193205343  Train loss: 1.16436022525
5 Test accuracy: 0.5286  Test loss: 1.38972
6 310 Train accuracy: 0.608427750472  Train loss: 1.10671260418
6 Test accuracy: 0.6163  Test loss: 1.12625
7 300 Train accuracy: 0.62535139468  Train loss: 1.059176832116310 Train accuracy: 0.62535139468  Train loss: 1.05917683211
7 Test accuracy: 0.6336  Test loss: 1.07312
8 310 Train accuracy: 0.641994634858  Train loss: 1.01702161133
8 Test accuracy: 0.6476  Test loss: 1.028
9 310 Train accuracy: 0.656397006769  Train loss: 0.978989842616
9 Test accuracy: 0.6467  Test loss: 1.03263
10 310 Train accuracy: 0.670840788633  Train loss: 0.940938851925
10 Test accuracy: 0.6403  Test loss: 1.06311
11 310 Train accuracy: 0.683254731545  Train loss: 0.908401865017
11 Test accuracy: 0.6157  Test loss: 1.19978
12 310 Train accuracy: 0.69563705135  Train loss: 0.8772487732082
12 Test accuracy: 0.6095  Test loss: 1.19249
13 310 Train accuracy: 0.705976790798  Train loss: 0.848525525374
Early stopping, best loss:  1.028
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'adam', 'batch_size': 256, 'learning_rate': 0.0015, 'ksize1': 5, 'filters1': 96, 'activation': 'elu', 'ksize3': 4, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 0, 'momentum': 0.9, 'filters2': 48}
1 154 Train accuracy: 0.455252902848  Train loss: 1.58493540968
1 Test accuracy: 0.5138  Test loss: 1.38897
2 154 Train accuracy: 0.533073910645  Train loss: 1.33625580158
2 Test accuracy: 0.5576  Test loss: 1.29266
3 154 Train accuracy: 0.589957364968  Train loss: 1.17341047242
3 Test accuracy: 0.6444  Test loss: 1.01092
4 154 Train accuracy: 0.624235668353  Train loss: 1.07670927261
4 Test accuracy: 0.6555  Test loss: 1.01554
5 150 Train accuracy: 0.657476358754  Train loss: 0.981298879215154 Train accuracy: 0.657476358754  Train loss: 0.981298879215
5 Test accuracy: 0.6887  Test loss: 0.910328
6 154 Train accuracy: 0.687974785055  Train loss: 0.901014026432
6 Test accuracy: 0.6525  Test loss: 1.0602
7 154 Train accuracy: 0.712618106482  Train loss: 0.833382792011
7 Test accuracy: 0.6792  Test loss: 1.00135
8 154 Train accuracy: 0.734157851764  Train loss: 0.771316996376
8 Test accuracy: 0.7033  Test loss: 0.910308
9 154 Train accuracy: 0.752887393747  Train loss: 0.718855348608
9 Test accuracy: 0.6714  Test loss: 1.17243
10 154 Train accuracy: 0.770205657823  Train loss: 0.671458637927
10 Test accuracy: 0.6823  Test loss: 1.19186
11 154 Train accuracy: 0.786143806074  Train loss: 0.626498006381
11 Test accuracy: 0.6876  Test loss: 1.25355
12 154 Train accuracy: 0.800491003054  Train loss: 0.586144522453
12 Test accuracy: 0.6812  Test loss: 1.25387
13 154 Train accuracy: 0.813699908964  Train loss: 0.549997791239
Early stopping, best loss:  0.910308
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 100, 'optimizer': 'rmsprop', 'batch_size': 64, 'learning_rate': 0.0015, 'ksize1': 5, 'filters1': 32, 'activation': 'lrelu', 'ksize3': 5, 'ksize2': 5, 'filters3': 96, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 96}
1 623 Train accuracy: 0.455384630859  Train loss: 1.46332220554
1 Test accuracy: 0.4413  Test loss: 1.57993
2 623 Train accuracy: 0.538461557478  Train loss: 1.26759196523
2 Test accuracy: 0.6025  Test loss: 1.14223
3 623 Train accuracy: 0.587487198412  Train loss: 1.14256919305
3 Test accuracy: 0.555  Test loss: 1.34693
4 623 Train accuracy: 0.621384633705  Train loss: 1.05528344572
4 Test accuracy: 0.6208  Test loss: 1.17022
5 623 Train accuracy: 0.651938478887  Train loss: 0.976881775379
5 Test accuracy: 0.632  Test loss: 1.15605
6 623 Train accuracy: 0.67312822178  Train loss: 0.9188723546273
6 Test accuracy: 0.5951  Test loss: 1.37202
7 623 Train accuracy: 0.693538477378  Train loss: 0.866675439903
7 Test accuracy: 0.6641  Test loss: 1.11752
8 623 Train accuracy: 0.710307707451  Train loss: 0.822083304375
8 Test accuracy: 0.679  Test loss: 1.07985
9 623 Train accuracy: 0.725264971753  Train loss: 0.779104506175
9 Test accuracy: 0.6653  Test loss: 1.11119
10 623 Train accuracy: 0.738830783099  Train loss: 0.740411006689
10 Test accuracy: 0.6834  Test loss: 1.1541
11 600 Train accuracy: 0.751104908396  Train loss: 0.707396330021623 Train accuracy: 0.751104908396  Train loss: 0.707396330021
11 Test accuracy: 0.6853  Test loss: 1.182
12 600 Train accuracy: 0.761692320481  Train loss: 0.675937940031623 Train accuracy: 0.761692320481  Train loss: 0.675937940031
12 Test accuracy: 0.6725  Test loss: 1.31166
13 623 Train accuracy: 0.772213030022  Train loss: 0.646205754234
Early stopping, best loss:  1.07985
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 32, 'learning_rate': 0.003, 'ksize1': 5, 'filters1': 96, 'activation': 'relu', 'ksize3': 4, 'ksize2': 4, 'filters3': 64, 'patch_reduction': 2, 'momentum': 0.99, 'filters2': 64}
1 1248 Train accuracy: 0.492727280259  Train loss: 1.40204974413
1 Test accuracy: 0.5077  Test loss: 1.37828
2 1248 Train accuracy: 0.567272733748  Train loss: 1.21352831423
2 Test accuracy: 0.5542  Test loss: 1.27794
3 1248 Train accuracy: 0.614343441129  Train loss: 1.09793996294
3 Test accuracy: 0.6268  Test loss: 1.0948
4 1248 Train accuracy: 0.645606067032  Train loss: 1.01585320964
4 Test accuracy: 0.668  Test loss: 0.964968
5 1248 Train accuracy: 0.67696970427  Train loss: 0.9333988934765
5 Test accuracy: 0.6803  Test loss: 0.970075
6 1248 Train accuracy: 0.702424250146  Train loss: 0.859957529902
6 Test accuracy: 0.696  Test loss: 0.926162
7 1248 Train accuracy: 0.722251090237  Train loss: 0.805403196088
7 Test accuracy: 0.7093  Test loss: 0.873374
8 1248 Train accuracy: 0.739393947348  Train loss: 0.757868852913
8 Test accuracy: 0.6872  Test loss: 0.978835
9 1248 Train accuracy: 0.752727280524  Train loss: 0.721052521798
9 Test accuracy: 0.7019  Test loss: 0.936568
10 1248 Train accuracy: 0.766242432177  Train loss: 0.686511354208
10 Test accuracy: 0.7031  Test loss: 0.983571
11 1248 Train accuracy: 0.779228658514  Train loss: 0.651754092899
11 Test accuracy: 0.6864  Test loss: 1.06602
12 1248 Train accuracy: 0.789242432465  Train loss: 0.621505219874
Early stopping, best loss:  0.873374
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 125, 'optimizer': 'nesterov', 'batch_size': 64, 'learning_rate': 0.002, 'ksize1': 5, 'filters1': 96, 'activation': 'relu', 'ksize3': 4, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 1, 'momentum': 0.99, 'filters2': 64}
1 623 Train accuracy: 0.45846155256  Train loss: 1.491717653273
1 Test accuracy: 0.4457  Test loss: 1.58997
2 623 Train accuracy: 0.540923095196  Train loss: 1.27675138831
2 Test accuracy: 0.5816  Test loss: 1.19482
3 623 Train accuracy: 0.58851283898  Train loss: 1.149691985453
3 Test accuracy: 0.6246  Test loss: 1.07153
4 623 Train accuracy: 0.622307710275  Train loss: 1.06182020962
4 Test accuracy: 0.6242  Test loss: 1.09557
5 623 Train accuracy: 0.645169248283  Train loss: 1.00319407272
5 Test accuracy: 0.645  Test loss: 1.03369
6 600 Train accuracy: 0.664615401576  Train loss: 0.948414717515623 Train accuracy: 0.664615401576  Train loss: 0.948414717515
6 Test accuracy: 0.6342  Test loss: 1.05424
7 623 Train accuracy: 0.68307693937  Train loss: 0.9002526043146
7 Test accuracy: 0.615  Test loss: 1.17004
8 623 Train accuracy: 0.699153861813  Train loss: 0.860234953314
8 Test accuracy: 0.6534  Test loss: 1.04636
9 600 Train accuracy: 0.714188049204  Train loss: 0.820682426691623 Train accuracy: 0.714188049204  Train loss: 0.820682426691
9 Test accuracy: 0.6501  Test loss: 1.09785
10 623 Train accuracy: 0.728307706684  Train loss: 0.783186449826
10 Test accuracy: 0.6806  Test loss: 0.998204
11 600 Train accuracy: 0.741202810976  Train loss: 0.748152696599623 Train accuracy: 0.741202810976  Train loss: 0.748152696599
11 Test accuracy: 0.6802  Test loss: 0.999456
12 600 Train accuracy: 0.75276924399  Train loss: 0.7170429464684623 Train accuracy: 0.75276924399  Train loss: 0.717042946468
12 Test accuracy: 0.6568  Test loss: 1.14526
13 623 Train accuracy: 0.763739657654  Train loss: 0.688456474588
13 Test accuracy: 0.6632  Test loss: 1.1126
14 623 Train accuracy: 0.773582429779  Train loss: 0.661880134983
14 Test accuracy: 0.691  Test loss: 1.02621
15 623 Train accuracy: 0.784451293687  Train loss: 0.632615184784
Early stopping, best loss:  0.998204
Training Model with following hyperparameters:
{'full_hidd1': 125, 'full_hidd2': 80, 'optimizer': 'adam', 'batch_size': 64, 'learning_rate': 0.002, 'ksize1': 4, 'filters1': 32, 'activation': 'relu', 'ksize3': 4, 'ksize2': 3, 'filters3': 96, 'patch_reduction': 2, 'momentum': 0.95, 'filters2': 64}
1 623 Train accuracy: 0.516307708025  Train loss: 1.38336248875
1 Test accuracy: 0.5371  Test loss: 1.29178
2 623 Train accuracy: 0.57138463378  Train loss: 1.201029120682
2 Test accuracy: 0.5852  Test loss: 1.16693
3 623 Train accuracy: 0.61743591547  Train loss: 1.081467441722
3 Test accuracy: 0.6327  Test loss: 1.06483
4 623 Train accuracy: 0.650000017285  Train loss: 0.990026459098
4 Test accuracy: 0.6629  Test loss: 0.958692
5 623 Train accuracy: 0.67716924715  Train loss: 0.9164506056319
5 Test accuracy: 0.6565  Test loss: 0.987663
6 623 Train accuracy: 0.700923092365  Train loss: 0.854429657261
6 Test accuracy: 0.5766  Test loss: 1.29068
7 623 Train accuracy: 0.72210990463  Train loss: 0.7967117287432
7 Test accuracy: 0.6476  Test loss: 1.09535
8 623 Train accuracy: 0.741076936722  Train loss: 0.745262564048
8 Test accuracy: 0.678  Test loss: 0.984954
9 623 Train accuracy: 0.756512833436  Train loss: 0.702025067608
Early stopping, best loss:  0.958692
Training Model with following hyperparameters:
{'full_hidd1': 60, 'full_hidd2': 100, 'optimizer': 'adam', 'batch_size': 128, 'learning_rate': 0.001, 'ksize1': 3, 'filters1': 64, 'activation': 'relu', 'ksize3': 5, 'ksize2': 3, 'filters3': 64, 'patch_reduction': 0, 'momentum': 0.95, 'filters2': 48}
1 310 Train accuracy: 0.537865242133  Train loss: 1.32324489263
1 Test accuracy: 0.5048  Test loss: 1.37785
2 310 Train accuracy: 0.597793681117  Train loss: 1.14690578442
2 Test accuracy: 0.5036  Test loss: 1.44896
3 310 Train accuracy: 0.640230572376  Train loss: 1.02669248673
3 Test accuracy: 0.6441  Test loss: 1.00156
4 310 Train accuracy: 0.666368516019  Train loss: 0.955371933488
4 Test accuracy: 0.6383  Test loss: 1.06123
5 310 Train accuracy: 0.688014311515  Train loss: 0.894868324353
5 Test accuracy: 0.6006  Test loss: 1.21398
6 310 Train accuracy: 0.708308486984  Train loss: 0.837254726352
6 Test accuracy: 0.6898  Test loss: 0.896807
7 310 Train accuracy: 0.726211771533  Train loss: 0.789039545989
7 Test accuracy: 0.6808  Test loss: 0.944838
8 310 Train accuracy: 0.742620749256  Train loss: 0.741297009186
8 Test accuracy: 0.6996  Test loss: 0.920914
9 310 Train accuracy: 0.756708405721  Train loss: 0.701008622463
9 Test accuracy: 0.6736  Test loss: 1.06946
10 310 Train accuracy: 0.76964817987  Train loss: 0.6648272751629
10 Test accuracy: 0.7002  Test loss: 0.979367
11 310 Train accuracy: 0.781482082772  Train loss: 0.631765787731
Early stopping, best loss:  0.896807
Training Model with following hyperparameters:
{'full_hidd1': 100, 'full_hidd2': 80, 'optimizer': 'nesterov', 'batch_size': 128, 'learning_rate': 0.0015, 'ksize1': 4, 'filters1': 32, 'activation': 'lrelu', 'ksize3': 3, 'ksize2': 5, 'filters3': 128, 'patch_reduction': 2, 'momentum': 0.9, 'filters2': 128}
1 310 Train accuracy: 0.431723316702  Train loss: 1.58174768778
1 Test accuracy: 0.5094  Test loss: 1.38247
2 310 Train accuracy: 0.508646392765  Train loss: 1.37356027502
2 Test accuracy: 0.5637  Test loss: 1.21329
3 310 Train accuracy: 0.559133372819  Train loss: 1.24854553663
3 Test accuracy: 0.5876  Test loss: 1.1623
4 310 Train accuracy: 0.592725103148  Train loss: 1.15738262351
4 Test accuracy: 0.5715  Test loss: 1.22155
5 310 Train accuracy: 0.620035776152  Train loss: 1.08066113829
5 Test accuracy: 0.6004  Test loss: 1.16128
6 310 Train accuracy: 0.648181275202  Train loss: 1.00501020525
6 Test accuracy: 0.5785  Test loss: 1.27352
7 310 Train accuracy: 0.670585228437  Train loss: 0.947621909144
7 Test accuracy: 0.5422  Test loss: 1.44701
8 310 Train accuracy: 0.693202145613  Train loss: 0.892398470018
8 Test accuracy: 0.6214  Test loss: 1.17507
9 310 Train accuracy: 0.71178691987  Train loss: 0.8445207835773
9 Test accuracy: 0.637  Test loss: 1.12809
10 310 Train accuracy: 0.728384018575  Train loss: 0.799778520144
10 Test accuracy: 0.6435  Test loss: 1.12751
11 310 Train accuracy: 0.744565511656  Train loss: 0.757046722247
11 Test accuracy: 0.6243  Test loss: 1.24456
12 310 Train accuracy: 0.759689922659  Train loss: 0.717245032963
12 Test accuracy: 0.6422  Test loss: 1.22417
13 310 Train accuracy: 0.774643363817  Train loss: 0.678645821189
13 Test accuracy: 0.6304  Test loss: 1.32182
14 310 Train accuracy: 0.787332820164  Train loss: 0.644556879015 Train accuracy: 0.787332820164  Train loss: 0.644556879015
14 Test accuracy: 0.639  Test loss: 1.34188
15 310 Train accuracy: 0.799920490269  Train loss: 0.610748880074
Early stopping, best loss:  1.12751