IST 597 Deep Learning Tensorflow Tutorial. -- Feed Forward Neural Network

Transcription

1 IST 597 Deep Learning Tensorflow Tutorial -- Feed Forward Neural Network September 19, 2018 Dataset Introduction The MNIST database (Modified National Institute of Standards and Technology database) is a large database of handwritten digits that is commonly used for training various image processing systems. The MNIST database contains 60,000 training images and 10,000 testing images. The black and white images from NIST were normalized to fit into a 28x28 pixel bounding box and anti-aliased, which introduced grayscale levels. Models Example 1 Softmax Regression import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import matplotlib.pyplot as plt input_dim = 784 output_dim = 10 # load data mnist = input_data.read_data_sets("mnist_data/", one_hot=true) trx, try, tex, tey = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels X = tf.placeholder("float", [None, input_dim]) Y = tf.placeholder("float", [None, output_dim]) # define model w = tf.variable(tf.random_normal([input_dim, output_dim], stddev=0.01)) b = tf.variable(tf.zeros([output_dim]) + 0.1) py_x = tf.matmul(x, w) + b

2 # define loss loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=y)) train_op = tf.train.gradientdescentoptimizer(0.05).minimize(loss) predict_op = tf.argmax(py_x, axis = 1) y_target = tf.argmax(y, axis = 1) accuracy = tf.reduce_mean(tf.cast(tf.equal(predict_op, y_target),tf.float32)) loss_vec = [] train_accuracy = [] test_accuracy = [] # Start Training with tf.session() as sess: tf.global_variables_initializer().run() for i in range(100): for start, end in zip(range(0, len(trx), 128), range(128, len(trx)+1, 128)): sess.run(train_op, feed_dict = {X: trx[start:end], Y: try[start:end]}) temp_loss = sess.run(loss, feed_dict = {X: trx, Y: try}) loss_vec.append(temp_loss) train_acc_temp = sess.run(accuracy, feed_dict = {X: trx, Y:trY}) train_accuracy.append(train_acc_temp) test_acc_temp = sess.run(accuracy, feed_dict = {X: tex, Y:teY}) test_accuracy.append(test_acc_temp) print(i, test_acc_temp) # Plot Figures plt.plot(train_accuracy) plt.plot(test_accuracy) plt.title('accuracy on training dataset and Test dataset') plt.ylabel('accuracy') plt.legend(['training accuracy','testing accuracy']) plt.ylim(0.85,0.95) plt.legend(loc='lower right') plt.plot(loss_vec) plt.title('loss on training dataset')

3

4 The performance of softmax regression is about 92.5%. Example 2 Multilayer Perceptron import tensorflow as tf from tensorflow.examples.tutorials.mnist import input_data import matplotlib.pyplot as plt input = 784 hidden = 625 output = 10 # define model def model(x, w_h, w_o, b_1, b_2): h = tf.nn.sigmoid(tf.matmul(x, w_h)+b_1) return tf.matmul(h, w_o)+b_2 # load data mnist = input_data.read_data_sets("mnist_data/", one_hot=true) trx, try, tex, tey = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels X = tf.placeholder("float", [None, 784]) Y = tf.placeholder("float", [None, 10]) # Initial parameters w_h = tf.variable(tf.random_normal([input, hidden],stddev=0.01)) w_o = tf.variable(tf.random_normal([hidden, output],stddev=0.01)) b_1 = tf.variable(tf.zeros([hidden]) + 0.1) b_2 = tf.variable(tf.zeros([output]) + 0.1) py_x = model(x, w_h, w_o, b_1, b_2) # define loss loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=y)) train_op = tf.train.gradientdescentoptimizer(0.05).minimize(loss) predict_op = tf.argmax(py_x, 1) y_target = tf.argmax(y, 1) accuracy = tf.reduce_mean(tf.cast(tf.equal(predict_op, y_target),tf.float32)) loss_vec = [] train_accuracy = [] test_accuracy = []

5 # Start Training with tf.session() as sess: tf.global_variables_initializer().run() for i in range(100): for start, end in zip(range(0, len(trx), 128), range(128, len(trx)+1, 128)): sess.run(train_op, feed_dict={x: trx[start:end], Y: try[start:end]}) temp_loss = sess.run(loss, feed_dict = {X: trx, Y: try}) loss_vec.append(temp_loss) train_acc_temp = sess.run(accuracy, feed_dict = {X: trx, Y:trY}) train_accuracy.append(train_acc_temp) test_acc_temp = sess.run(accuracy, feed_dict = {X: tex, Y:teY}) test_accuracy.append(test_acc_temp) print(i, test_acc_temp) # Plot figures plt.plot(train_accuracy) plt.plot(test_accuracy) plt.title('accuracy on training dataset and Test dataset') plt.ylabel('accuracy') plt.legend(['training accuracy','testing accuracy']) plt.ylim(0.85,1.0) plt.legend(loc='lower right') plt.plot(loss_vec) plt.title('loss on training dataset') The performance of multilayer perceptron is about 95.4%.

6