This document describes an implementation of backpropagation algorithm in C code to understand the backpropagation algorithm through programming. It also explains how it can be applied to a real-world case of handwritten digit recognition using TensorFlow code. The neural network code can be used to gain experience by adjusting values and observing their effects on the network.

1. 오류 역전파 알고리즘 구현
딥러닝을 위한 신경망 기초
nonezerok@gmail.com
프로그램 코드를 통해 오류역전파알고리즘에 대한 이해를 높일 수 있습니다.
여기서 제시되는 C 코드는 오류역전파알고리즘을 이해하는 데 목적을 두어 최대한 간결하게 작성되어 있습니다.
또한 Tensorflow 코드로 제공된 필기체 숫자인식을 통해 실제 사례에 어떻게 적용되는지를 살펴볼 수 있습니다.
신경망은 주어진 데이터에 따라 학습자가 조정해야 할 값들이 있습니다.
이 값들을 변경해 보면서 조정값들에 대한 경험치를 쌓는데 본 코드를 활용할 수 있습니다.

2. 2
MLP for XOR pattern
in C

3. 3
 =
1
1
 = 1
 =
0
1
 = 0
 =
0
0
 = 0
 =
1
0
 = 0
Input Layer Hidden Layer Output Layer
1 1
Net0
Net1
Net2
0
1
2
w00
w10
w11
w21
w20
w02
0
w12
w22
x1
x2
y
w01
Weight[3][3]
out
포물선으로 구분

4. 4
/* ========================================== *
* Filename: bpnet.h
* Author: James Matthews.
* This is a tiny neural network that uses
* back propagation for weight adjustment.
* ========================================== */
#define BP_LEARNING(float)(0.5)// The learning coefficient.
class CBPNet{
public:
CBPNet();
~CBPNet() {};
float Train(float, float, float);
float Run(float, float);
private:
float m_fWeights[3][3];// Weights for the 3 neurons.
float Sigmoid(float);// The sigmoid function.
};

5. 5
CBPNet::CBPNet()
{
srand((unsigned)(time(NULL)));
for (inti=0;i<3;i++) {
for (intj=0;j<3;j++) {
m_fWeights[i][j] = (float)(rand())/(32767/2) -1;
}
}
}

6. 6
float CBPNet::Train(floati1, float i2, float y)
{
float net1, net2, i3, i4, out;
net1 = 1*m_fWeights[0][0]+i1*m_fWeights[1][0]+i2*m_fWeights[2][0];
net2 = 1*m_fWeights[0][1]+i1*m_fWeights[1][1]+i2*m_fWeights[2][1];
i3 = Sigmoid(net1);
i4 = Sigmoid(net2);
net1 = 1*m_fWeights[0][2]+i3*m_fWeights[1][2]+i4*m_fWeights[2][2];
out = Sigmoid(net1);

7. 7
float deltas[3];
deltas[2] = out*(1-out)*(y-out);
deltas[1] = i4*(1-i4)*(m_fWeights[2][2])*(deltas[2]);
deltas[0] = i3*(1-i3)*(m_fWeights[1][2])*(deltas[2]);
 ←  1 −   − 
 ←  1 −   ,


Input Layer Hidden Layer Output Layer
1 1
Net0
Net1
Net2
0
1
2
w00
w10
w11
w21
w20
w02
0
w12
w22
x1
x2
y
w01
Weight[3][3]
out

8. float v1 = i1, v2 = i2;
for(int i=0;i<3;i++) {
// Change the values for the output layer, if necessary.
if (i == 2) {
v1 = i3;
v2 = i4;
}
m_fWeights[0][i] += BP_LEARNING*1*deltas[i];
m_fWeights[1][i] += BP_LEARNING*v1*deltas[i];
m_fWeights[2][i] += BP_LEARNING*v2*deltas[i];
}
return out;
}
, ← , + ∆,
∆, = 
 =  when  is of the first layer

9. 9
float CBPNet::Sigmoid(floatnum)
{
return (float)(1/(1+exp(-num)));
}
float CBPNet::Run(floati1, float i2)
{
float net1, net2, i3, i4;
net1 = 1*m_fWeights[0][0]+i1*m_fWeights[1][0]+i2*m_fWeights[2][0];
net2 = 1*m_fWeights[0][1]+i1*m_fWeights[1][1]+i2*m_fWeights[2][1];
i3 = Sigmoid(net1);
i4 = Sigmoid(net2);
net1 = 1*m_fWeights[0][2]+i3*m_fWeights[1][2]+i4*m_fWeights[2][2];
return Sigmoid(net1);
}

10. 10
#define BPM_ITER2000
void main()
{
CBPNet bp;
for (inti=0;i<BPM_ITER;i++) {
bp.Train(0,0,0);
bp.Train(0,1,1);
bp.Train(1,0,1);
bp.Train(1,1,0);
}
cout<< "0,0 = " << bp.Run(0,0) << endl;
cout<< "0,1 = " << bp.Run(0,1) << endl;
cout<< "1,0 = " << bp.Run(1,0) << endl;
cout<< "1,1 = " << bp.Run(1,1) << endl;
}

11. 11
1 1
Net0
Net1
Net2
0
1
2
w00
w10
w11
w21
w20
w02
0
w12
w22
x1
x2
y1
w01
out
Net3 y2
out
출력 층의 노드를 2개로 확장 구현 스스로!

12. 12
MLP on MNIST
in Tensorflow

13. 13
http://yann.lecun.com/exdb/mnist/
28
28
28X28=768
• 5,5000개 학습 데이터
• 10,000개 테스트 데이터
MNIST

14. 14
https://ml4a.github.io/ml4a/neural_networks/
0~1사이로 변경 후 입력

15. 15
3 layers perceptron
https://ml4a.github.io/demos/forward_pass_mnist/
8번 뉴런 (노드)에서 가장
큰 값을 출력
8과 유사한 것 중 하나

16. 16
'
' https://github.com/rickiepark/tfk-notebooks
'
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
plt.imshow(mnist.train.images[100].reshape([28, 28]),
cmap=plt.get_cmap('gray_r'))
x = tf.placeholder("float32", [None, 784])
y = tf.placeholder("float32", shape=[None, 10])
W1 = tf.Variable(tf.truncated_normal([784, 100], stddev=0.1))
b1 = tf.Variable(tf.constant(0.1, shape=[100]))

17. 17
t = tf.sigmoid(tf.matmul(x,W1) + b1)
W2 = tf.Variable(tf.truncated_normal([100, 10], stddev=0.1))
b2 = tf.Variable(tf.constant(0.1, shape=[10]))
z = tf.matmul(t,W2) + b2
y_hat = tf.nn.softmax(z)
loss = tf.losses.softmax_cross_entropy(y, z)
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_hat,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
sess = tf.Session()
sess.run(tf.global_variables_initializer())

18. 18
costs = []
for i in range(1000):
x_data, y_data = mnist.train.next_batch(100)
_, cost = sess.run([train, loss], feed_dict={x: x_data, y: y_data})
costs.append(cost)
plt.plot(costs)
for i in range(5):
plt.imshow(mnist.test.images[i].reshape([28, 28]),
cmap=plt.get_cmap('gray_r'))
plt.show()
print(sess.run(tf.argmax(y_hat,1),
feed_dict={x: mnist.test.images[i].reshape([1,784])}))