AlexNet卷积神经网络实现

AlexNet特点

• 使用ReLU作为**函数，解决了sigmoid在网络较深时的梯度弥散问题
• 使用Dropout随机忽略一部分神经元，以避免过拟合
• 使用重叠的maxpool，避免argpool的模糊化效果
• 提出LRN层，对局部神经元的活动创建竞争机制，使其中响应比较大的值变得相对更大，并抑制其它反馈较小的神经元，增强模型的泛化能力
• GTX 580*2 3GB
• 数据增强：随机从256256的原始图像中截取224224区域，并水平翻转，相当于增加了（256-224）^2*2=2048倍的数据量，大大减轻了过拟合

源代码

注释已经写得很清楚了，不再赘述~

from datetime import datetime
import math
import tensorflow as tf
import time

batch_size = 32 
num_batches = 100

def print_activations(t):
	# 打印网络结构
	print(t.op.name,'',t.get_shape().as_list())

def inference(images):
	parameters = []
	# conv1
	with tf.name_scope('conv1') as scope:
		#使用截断正态分布函数初始化卷积核参数 卷积核尺寸11*11 通道3 卷积核数量64
		kernel = tf.Variable(tf.truncated_normal([11,11,3,64],dtype=tf.float32,stddev=1e-1),name='weights')
		# 完成对图像的卷积操作 strides步长4*4（即图像上每4*4区域只取样一次）
		conv = tf.nn.conv2d(images,kernel,[1,4,4,1],padding='SAME')
		# 初始化biases为0
		biases = tf.Variable(tf.constant(0.0,shape=[64],dtype=tf.float32),trainable=True,name='biases')
		bias = tf.nn.bias_add(conv,biases)
		# ReLU**函数对结果进行非线性处理
		conv1 = tf.nn.relu(bias,name=scope)
		print_activations(conv1)
		# 将可训练参数（kernel,biases）添加至 parameters 
		parameters += [kernel,biases]
		# 自主选择是否选用LRN  depth_radius设为4
		lrn1 = tf.nn.lrn(conv1,4,bias=1.0,alpha=0.001/9,beta=0.75,name='lrn1')
		# 池化尺寸3*3（即将3*3大小的像素块降为1*1的像素） padding模式VALID,即取样是不能超过边框
		pool1 = tf.nn.max_pool(lrn1,ksize=[1,3,3,1],strides=[1,2,2,1],padding='VALID',name='pool1')
		# 打印输出结果pool1的结构
		print_activations(pool1)

	# conv2
	with tf.name_scope('conv2') as scope:
		# 卷积核尺寸5*5 通道64（上一层输出通道数，也就是上一层是卷积核数量）卷积核数量194
		kernel = tf.Variable(tf.truncated_normal([5,5,64,192],dtype=tf.float32,stddev=1e-1),name='weights')
		# 卷积步长1*1 padding模式为SAME:矩形周围补2个像素
		conv = tf.nn.conv2d(pool1,kernel,[1,1,1,1],padding='SAME')
		biases = tf.Variable(tf.constant(0.0,shape=[192],dtype=tf.float32),trainable=True,name='biases')
		bias = tf.nn.bias_add(conv,biases)
		conv2 = tf.nn.relu(bias,name=scope)
		print_activations(conv2)
		parameters += [kernel,biases]
		lrn2 = tf.nn.lrn(conv2,4,bias=1.0,alpha=0.001/9,beta=0.75,name='lrn2')
		pool2 = tf.nn.max_pool(lrn2,ksize=[1,3,3,1],strides=[1,2,2,1],padding='VALID',name='pool2')
		print_activations(pool2)		

	# conv3
	with tf.name_scope('conv3') as scope:
		# 卷积核尺寸3*3 输入通道数192 输出通道数384
		kernel = tf.Variable(tf.truncated_normal([3,3,192,384],dtype=tf.float32,stddev=1e-1),name='weights')
		# 卷积步长1*1  padding模式为SAME:矩形周围补1个像素
		conv = tf.nn.conv2d(pool2,kernel,[1,1,1,1],padding='SAME')
		biases = tf.Variable(tf.constant(0.0,shape=[384],dtype=tf.float32),trainable=True,name='biases')
		bias = tf.nn.bias_add(conv,biases)
		conv3 = tf.nn.relu(bias,name=scope)
		print_activations(conv3)
		parameters += [kernel,biases]

	# conv4
	with tf.name_scope('conv4') as scope:
		# 卷积核尺寸3*3 输入通道数384 输出通道数256
		kernel = tf.Variable(tf.truncated_normal([3,3,384,256],dtype=tf.float32,stddev=1e-1),name='weights')
		# 卷积步长1*1
		conv = tf.nn.conv2d(conv3,kernel,[1,1,1,1],padding='SAME')
		biases = tf.Variable(tf.constant(0.0,shape=[256],dtype=tf.float32),trainable=True,name='biases')
		bias = tf.nn.bias_add(conv,biases)
		conv4 = tf.nn.relu(bias,name=scope)
		print_activations(conv4)
		parameters += [kernel,biases]

	# conv5
	with tf.name_scope('conv5') as scope:
		# 卷积核尺寸3*3 输入通道数256 输出通道数256
		kernel = tf.Variable(tf.truncated_normal([3,3,256,256],dtype=tf.float32,stddev=1e-1),name='weights')
		# 卷积步长1*1
		conv = tf.nn.conv2d(conv4,kernel,[1,1,1,1],padding='SAME')
		biases = tf.Variable(tf.constant(0.0,shape=[256],dtype=tf.float32),trainable=True,name='biases')
		bias = tf.nn.bias_add(conv,biases)
		conv5 = tf.nn.relu(bias,name=scope)
		print_activations(conv5)
		parameters += [kernel,biases]
		# maxpool
		pool5 = tf.nn.max_pool(conv5,ksize=[1,3,3,1],strides=[1,2,2,1],padding='VALID',name='pool5')
		print_activations(pool5)

	# fc6
	
	# fc7
	
	# fc8
	
	return pool5,parameters

def time_tensorflow_run(session,target,info_string):
	# 预热轮数 给程序预热
	num_steps_burn_in = 10
	# 总时间
	total_duration = 0.0
	# 总时间的平方和 以计算方差
	total_duration_squared = 0.0

	for i in range(num_batches+num_steps_burn_in):
		start_time = time.time()
		_ = session.run(target)
		duration = time.time() - start_time
		if i >= num_steps_burn_in:
			if not i % 10:
				print('%s:step %d, duration = %.3f' % (datetime.now(),i - num_steps_burn_in,duration))
			total_duration += duration
			total_duration_squared += duration*duration
	# 计算每轮迭代的平均耗时mn
	mn = total_duration / num_batches
	vr = total_duration_squared /num_batches - mn*mn
	# 标准差
	sd = math.sqrt(vr)	
	print('%s: %s across %d steps, %.3f +/- %.3f sec / batch' % 
			(datetime.now(),info_string,num_batches,mn,sd))

def run_benchmark():
	with tf.Graph().as_default():
		image_size = 224
		images = tf.Variable(tf.random_normal([batch_size,image_size,image_size,3],
												dtype=tf.float32,
												stddev=1e-1))
		pool5,parameters = inference(images)
		init = tf.global_variables_initializer()
		sess = tf.Session()
		sess.run(init)

		time_tensorflow_run(sess,pool5,'Forward')
		objective = tf.nn.l2_loss(pool5)
		grad = tf.gradients(objective,parameters)
		time_tensorflow_run(sess,grad,"Forward-backward")

# 主函数
run_benchmark()

运行结果

cpu:i5 2.6GHz

GPU:Nvida Quardro 2000 5GB
(教材上GTX 1080 跑的是0.026)