tensorflow tips

发表于
字数统计: 887 | 阅读时长 ≈ 4

tf.InteractiveSession() 与 tf.Session()

1
2
3
4
5
6
7
sess = tf.InteractiveSession()
a = tf.constant(5.0)
b = tf.constant(6.0)
c = a * b
# We can just use 'c.eval' without passing 'sess'
print(c.eval())
sess.close()

而tf.Session()前必须构建好图，再创建session

1
2
3
4
5
6
7
8
# Build a graph
a = tf.constant(5.0)
b = tf.constant(6.0)
c = a * b
# Launch the graph in a Session
sess = tf.Session()
# Evaluate the tensor 'c'
print(sess.run(c))

tf.app.flags 与 python3 absl.flags

1
2
3
4
5
6
7
8
9
10
from absl import app

FLAGS = flags.FLAGS
flags.DEFINE_string('gpu', None, 'comma separated list of GPU to use')

def main(argv):
    print(FLAGS.gpu)

if __name__ == '__main__':
    app.run(main)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import tensorflow as tf

flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_float(‘learning_rate’, 0.01, ‘Initial learning rate.’)
flags.DEFINE_integer(‘max_steps’, 2000, ‘Number of steps to run trainer.’)
flags.DEFINE_string(‘train_dir’, ‘data’, ‘Directory to put the training data.’)
flags.DEFINE_boolean(‘fake_data’, False, ‘If true, uses fake data ‘
‘for unit testing.’)

def main():
    print(FLAGS.learning_rate)

if __name__ == '__main__':
    tf.app.run()
1
2
3
4
5
6
7
8
9
10
11
12
import argparse

parser = argparse.ArgumentParser(description="Demo")
parser.add_argument('-y', '--year', default='2019')
parser.add_argument('-m', '--month', default='Sep')
args = parser.parse_args()

def main():
    print(args)

if __name__ == '__main__':
    main()

tf.train.ExponentialMovingAverage

Some training algorithms, such as GradientDescent and Momentum often benefit from maintaining a moving average of variables during optimization. Using the moving averages for evaluations often improve results significantly.

tf.train.ExponentialMovingAverage.__init__(self, decay, num_updates=None, zero_debias=False, name="ExponentialMovingAverage"):

参数的移动平均验证效果优于最终训练的结果。

$shadow_variable$ 最初为参数的 $copy$ ，上次为指数衰减， $decay$ 为衰减速率，一般为 $0.9, 0.99$ 等接近 $1$ 的数。

一般来讲，decay越大模型越稳定，因为参数更新更慢，趋于稳定。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
import tensorflow as tf

# 定义一个32位浮点数的变量，初始值位0.0
v1 = tf.Variable(dtype=tf.float32, initial_value=0.)
# 衰减率decay，初始值位0.99
decay = 0.99
# 定义num_updates，同样，初始值位0
num_updates = tf.Variable(0, trainable=False)
# 定义滑动平均模型的类，将衰减率decay和num_updates传入。
ema = tf.train.ExponentialMovingAverage(decay=decay, num_updates=num_updates)
# 定义更新变量列表
update_var_list = [v1]
# 使用滑动平均模型
ema_apply = ema.apply(update_var_list) # return an operation that updates the moving averages

with tf.Session() as sess:
   # 初始化全局变量
   sess.run(tf.global_variables_initializer())

   print(sess.run([v1, ema.average(v1)]))
   # shadow_variable = variable = 0.
   sess.run(tf.assign(v1, 5))
   sess.run(ema_apply)
   print(sess.run([v1, ema.average(v1)]))

可参考吴恩达 $deeeplearning.ai$ 指数加权平均。

tf.add_to_collection 与 tf.get_collection

1
2
3
4
5
6
7
8
9
10
11
12
import tensorflow as tf;  

v1 = tf.get_variable(name='v1', shape=[1], initializer=tf.constant_initializer(0))
tf.add_to_collection('loss', v1)
v2 = tf.get_variable(name='v2', shape=[1], initializer=tf.constant_initializer(2))
tf.add_to_collection('loss', v2)

with tf.Session() as sess:
	sess.run(tf.initialize_all_variables())

	print tf.get_collection('loss')
	print sess.run(tf.add_n(tf.get_collection('loss')))

例如在使用L2正则化，网路各层变量需计算tf.nn.l2_loss，将其加入loss列表，最后创建操作求和。

tf.control_dependencies

1
2
3
4
with g.control_dependencies([a, b, c]):
#’d’和’e’只会在’a’,’b’和’c’运行之后运行
  d = ...
  e = ...

tf.name_scope 与 tf.variable_scope

知乎 tensorflow.name_scope如何理解

总结
1.tf.variable_scope 与 tf.get_variable 必须搭配使（全局 scope 除外）
2.tf.Variable 可以单独使用，也可以搭配 tf.name_scope 使用，给变量分类命名，模块化
3.tf.Variable 与 tf.variable_scope 搭配使用不伦不类，不是设计者的初衷

当前环境作用域可通过 tf.get_variable_scope 获取。

tensorflow.contrib.slim API

1
2
3
4
5
6
7
8
import tensorflow as tf

input = ...
with tf.variable_scope('conv1') as scope:
    kernel = tf.get_variable(name='weight', shape=[3, 3, 64, 128], initializer=tf.truncated_normal_initializer(stddev=5e-2, dtype=tf.float32), dtype=tf.float32)
    conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME')
    biases = tf.get_variable(name='biases', shape=[64], initializer=tf.constant_initializer(0.0), dtype=tf.float32)
    conv1 = tf.nn.relu(tf.nn.bias_add(conv, biases), name=scope.name)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# slim convolution 原型
convolution(inputs,
        num_outputs,
        kernel_size,
        stride=1,
        padding='SAME',
        data_format =None,
        rate = 1,
        activation_fn = nn.relu,
        normalizer_params=None,
        weights_initializer = initializers.xavier_initializer(),
        weights_regularizer = None,
        biases_initializer = init_ops.zeros_initializer(),
        biases_regularizer = None,
        reuse = None,
        variables_collections = None,
        outputs_collections = None,
        trainable = True,
        scope = None):
1
2
3
4
5
import tensorflow as tf
import tensorflow.contrib.slim as slim

input = ...
net = slim.conv2d(input, 128, [3, 3]. scope='conv')

slim简化更有利于复杂网络编程，slim更多内容请参考官方说明

Github tensorflow/contrib/slim说明

0%