1
\$\begingroup\$

I am new to TensorFlow and I would really appreciate if someone could look at my code to see whether things are done efficiently and suggest improvements.

This code works okay and achieves around 91.5% accuracy on test data. The only real issue I have is a low GPU usage during training reported by GPU-Z (27%). I am not sure whether this is a real problem or not.

def dense(x, size, scope):
    return tf.contrib.layers.fully_connected(x, size, activation_fn=None, scope=scope)

def dense_batch_relu_dropout(x, size, is_training, keep_prob, scope):
    with tf.variable_scope(scope):
        fc = dense(x, size, 'dense')
        fc_norm = tf.contrib.layers.batch_norm(
            fc, center=True, scale=True, is_training=is_training, scope='bn',
        )
        fc_relu = tf.nn.relu(fc_norm, 'relu')
        return tf.nn.dropout(fc_relu, keep_prob)

def train_deeper_better(train_data, train_labels, test_data, test_labels):
    """Same as 'train_deeper', but now with tf.contrib.data.Dataset input pipeline."""
    graph = tf.Graph()
    with graph.as_default():
        tf.set_random_seed(52)

        # dataset definition
        dataset = Dataset.from_tensor_slices({'x': train_data, 'y': train_labels})
        dataset = dataset.shuffle(buffer_size=20000)
        dataset = dataset.batch(256)
        iterator = dataset.make_initializable_iterator()
        sample = iterator.get_next()
        x = sample['x']
        y = sample['y']

        # actual computation graph
        keep_prob = tf.placeholder(tf.float32)
        is_training = tf.placeholder(tf.bool, name='is_training')

        fc1 = dense_batch_relu_dropout(x, 1024, is_training, keep_prob, 'fc1')
        fc2 = dense_batch_relu_dropout(fc1, 300, is_training, keep_prob, 'fc2')
        fc3 = dense_batch_relu_dropout(fc2, 50, is_training, keep_prob, 'fc3')
        logits = dense(fc3, NUM_CLASSES, 'logits')

        with tf.name_scope('accuracy'):
            accuracy = tf.reduce_mean(
                tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32),
            )
            accuracy_percent = 100 * accuracy

        with tf.name_scope('loss'):
            loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))

        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        with tf.control_dependencies(update_ops):
            # ensures that we execute the update_ops before performing the train_op
            # needed for batch normalization (apparently)
            train_op = tf.train.AdamOptimizer(learning_rate=1e-2, epsilon=1e-3).minimize(loss)

    with tf.Session(graph=graph) as sess:
        tf.global_variables_initializer().run()

        tf.logging.set_verbosity(tf.logging.FATAL)
        os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

        logger.info('Starting training...')
        step = 0
        epoch = 0
        while True:
            sess.run(iterator.initializer, feed_dict={})
            while True:
                step += 1
                try:
                    sess.run(train_op, feed_dict={keep_prob: 0.5, is_training: True})
                    if step % 1000 == 0:
                        l, ac = sess.run(
                            [loss, accuracy_percent],
                            feed_dict={keep_prob: 0.5, is_training: False},
                        )
                        logger.info('Minibatch loss: %f, accuracy: %.2f%%', l, ac)
                except tf.errors.OutOfRangeError:
                    logger.info('End of epoch #%d', epoch)
                    break

            # end of epoch
            train_l, train_ac = sess.run(
                [loss, accuracy_percent],
                feed_dict={x: train_data, y: train_labels, keep_prob: 1, is_training: False},
            )
            test_l, test_ac = sess.run(
                [loss, accuracy_percent],
                feed_dict={x: test_data, y: test_labels, keep_prob: 1, is_training: False},
            )
            logger.info('Train loss: %f, train accuracy: %.2f%%', train_l, train_ac)
            logger.info('Test loss: %f, test accuracy: %.2f%%', test_l, test_ac)

            epoch += 1
            logger.info('Starting new epoch #%d!', epoch)
\$\endgroup\$
  • 1
    \$\begingroup\$ Using Tensorflow 1.3.0, I'm getting the follow error: TypeError: Expected binary or unicode string, got {'x': array(..... Consider casting elements to a supported type. On the line "dataset = Dataset.from_tensor_slices({'x': train_data, 'y': train_labels})" \$\endgroup\$ – kissste Oct 11 '17 at 16:48

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.