From 350bbd382959acc30857b4bede76cba9665cc69b Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Fri, 25 Oct 2019 00:23:15 +0530 Subject: [PATCH 1/9] Create GAN.py --- neural_network/GAN.py | 397 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 neural_network/GAN.py diff --git a/neural_network/GAN.py b/neural_network/GAN.py new file mode 100644 index 000000000000..634d3b40cc24 --- /dev/null +++ b/neural_network/GAN.py @@ -0,0 +1,397 @@ +import numpy as np,sys,time +from sklearn.utils import shuffle +import matplotlib.pyplot as plt +import matplotlib +from tensorflow.examples.tutorials.mnist import input_data +import os +import numpy as np,sys +from sklearn.utils import shuffle +import matplotlib.pyplot as plt +import matplotlib.gridspec as gridspec + +random_numer = int(input("Please Input a Random Number to Seed")) + + +np.random.seed(random_numer) +def ReLu(x): + mask = (x>0) * 1.0 + return mask *x +def d_ReLu(x): + mask = (x>0) * 1.0 + return mask + +def arctan(x): + return np.arctan(x) +def d_arctan(x): + return 1 / (1 + x ** 2) + +def log(x): + return 1 / ( 1+ np.exp(-1*x)) +def d_log(x): + return log(x) * (1 - log(x)) + +def tanh(x): + return np.tanh(x) +def d_tanh(x): + return 1 - np.tanh(x) ** 2 + +def plot(samples): + fig = plt.figure(figsize=(4, 4)) + gs = gridspec.GridSpec(4, 4) + gs.update(wspace=0.05, hspace=0.05) + + for i, sample in enumerate(samples): + ax = plt.subplot(gs[i]) + plt.axis('off') + ax.set_xticklabels([]) + ax.set_yticklabels([]) + ax.set_aspect('equal') + plt.imshow(sample.reshape(28, 28), cmap='Greys_r') + + return fig + + + +# 1. Load Data and declare hyper +print('--------- Load Data ----------') +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +temp = mnist.test +images, labels = temp.images, temp.labels +images, labels = shuffle(np.asarray(images),np.asarray(labels)) +num_epoch = 10 +learing_rate = 0.00009 +G_input = 100 +hidden_input,hidden_input2,hidden_input3 = 128,256,346 +hidden_input4,hidden_input5,hidden_input6 = 480,560,686 + + + +print('--------- Declare Hyper Parameters ----------') +# 2. Declare Weights +D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.))) *0.002 +# D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +D_b1 = np.zeros(hidden_input) + +D_W2 = np.random.normal(size=(hidden_input,1),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +# D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.))) *0.002 +D_b2 = np.zeros(1) + + +G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b1 = np.zeros(hidden_input) + +G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b2 = np.zeros(hidden_input2) + +G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b3 = np.zeros(hidden_input3) + +G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b4 = np.zeros(hidden_input4) + +G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b5 = np.zeros(hidden_input5) + +G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b6 = np.zeros(hidden_input6) + +G_W7 = np.random.normal(size=(hidden_input6,784),scale=(1. / np.sqrt(hidden_input6 / 2.))) *0.002 +# G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.))) *0.002 +G_b7 = np.zeros(784) + +# 3. For Adam Optimzier +v1,m1 = 0,0 +v2,m2 = 0,0 +v3,m3 = 0,0 +v4,m4 = 0,0 + +v5,m5 = 0,0 +v6,m6 = 0,0 +v7,m7 = 0,0 +v8,m8 = 0,0 +v9,m9 = 0,0 +v10,m10 = 0,0 +v11,m11 = 0,0 +v12,m12 = 0,0 + +v13,m13 = 0,0 +v14,m14 = 0,0 + +v15,m15 = 0,0 +v16,m16 = 0,0 + +v17,m17 = 0,0 +v18,m18 = 0,0 + + +beta_1,beta_2,eps = 0.9,0.999,0.00000001 + +print('--------- Started Training ----------') +for iter in range(num_epoch): + + random_int = np.random.randint(len(images) - 5) + current_image = np.expand_dims(images[random_int],axis=0) + + # Func: Generate The first Fake Data + Z = np.random.uniform(-1., 1., size=[1, G_input]) + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + # Func: Forward Feed for Real data + Dl1_r = current_image.dot(D_W1) + D_b1 + Dl1_rA = ReLu(Dl1_r) + Dl2_r = Dl1_rA.dot(D_W2) + D_b2 + Dl2_rA = log(Dl2_r) + + # Func: Forward Feed for Fake Data + Dl1_f = current_fake_data.dot(D_W1) + D_b1 + Dl1_fA = ReLu(Dl1_f) + Dl2_f = Dl1_fA.dot(D_W2) + D_b2 + Dl2_fA = log(Dl2_f) + + # Func: Cost D + D_cost = -np.log(Dl2_rA) + np.log(1.0- Dl2_fA) + + # Func: Gradient + grad_f_w2_part_1 = 1/(1.0- Dl2_fA) + grad_f_w2_part_2 = d_log(Dl2_f) + grad_f_w2_part_3 = Dl1_fA + grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2) + grad_f_b2 = grad_f_w2_part_1 * grad_f_w2_part_2 + + grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T) + grad_f_w1_part_2 = d_ReLu(Dl1_f) + grad_f_w1_part_3 = current_fake_data + grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2) + grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2 + + grad_r_w2_part_1 = - 1/Dl2_rA + grad_r_w2_part_2 = d_log(Dl2_r) + grad_r_w2_part_3 = Dl1_rA + grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2) + grad_r_b2 = grad_r_w2_part_1 * grad_r_w2_part_2 + + grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T) + grad_r_w1_part_2 = d_ReLu(Dl1_r) + grad_r_w1_part_3 = current_image + grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2) + grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2 + + grad_w1 =grad_f_w1 + grad_r_w1 + grad_b1 =grad_f_b1 + grad_r_b1 + + grad_w2 =grad_f_w2 + grad_r_w2 + grad_b2 =grad_f_b2 + grad_r_b2 + + # ---- Update Gradient ---- + m1 = beta_1 * m1 + (1 - beta_1) * grad_w1 + v1 = beta_2 * v1 + (1 - beta_2) * grad_w1 ** 2 + + m2 = beta_1 * m2 + (1 - beta_1) * grad_b1 + v2 = beta_2 * v2 + (1 - beta_2) * grad_b1 ** 2 + + m3 = beta_1 * m3 + (1 - beta_1) * grad_w2 + v3 = beta_2 * v3 + (1 - beta_2) * grad_w2 ** 2 + + m4 = beta_1 * m4 + (1 - beta_1) * grad_b2 + v4 = beta_2 * v4 + (1 - beta_2) * grad_b2 ** 2 + + D_W1 = D_W1 - (learing_rate / (np.sqrt(v1 /(1-beta_2) ) + eps)) * (m1/(1-beta_1)) + D_b1 = D_b1 - (learing_rate / (np.sqrt(v2 /(1-beta_2) ) + eps)) * (m2/(1-beta_1)) + + D_W2 = D_W2 - (learing_rate / (np.sqrt(v3 /(1-beta_2) ) + eps)) * (m3/(1-beta_1)) + D_b2 = D_b2 - (learing_rate / (np.sqrt(v4 /(1-beta_2) ) + eps)) * (m4/(1-beta_1)) + + # Func: Forward Feed for G + Z = np.random.uniform(-1., 1., size=[1, G_input]) + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + Dl1 = current_fake_data.dot(D_W1) + D_b1 + Dl1_A = ReLu(Dl1) + Dl2 = Dl1_A.dot(D_W2) + D_b2 + Dl2_A = log(Dl2) + + # Func: Cost G + G_cost = -np.log(Dl2_A) + + # Func: Gradient + grad_G_w7_part_1 = ((-1/Dl2_A) * d_log(Dl2).dot(D_W2.T) * (d_ReLu(Dl1))).dot(D_W1.T) + grad_G_w7_part_2 = d_log(Gl7) + grad_G_w7_part_3 = Gl6A + grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_1) + grad_G_b7 = grad_G_w7_part_1 * grad_G_w7_part_2 + + grad_G_w6_part_1 = (grad_G_w7_part_1 * grad_G_w7_part_2).dot(G_W7.T) + grad_G_w6_part_2 = d_ReLu(Gl6) + grad_G_w6_part_3 = Gl5A + grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2) + grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2) + + grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T) + grad_G_w5_part_2 = d_tanh(Gl5) + grad_G_w5_part_3 = Gl4A + grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2) + grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2) + + grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T) + grad_G_w4_part_2 = d_ReLu(Gl4) + grad_G_w4_part_3 = Gl3A + grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2) + grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2) + + grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T) + grad_G_w3_part_2 = d_arctan(Gl3) + grad_G_w3_part_3 = Gl2A + grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2) + grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2) + + grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T) + grad_G_w2_part_2 = d_ReLu(Gl2) + grad_G_w2_part_3 = Gl1A + grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2) + grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2) + + grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T) + grad_G_w1_part_2 = d_arctan(Gl1) + grad_G_w1_part_3 = Z + grad_G_w1 = grad_G_w1_part_3.T.dot(grad_G_w1_part_1 * grad_G_w1_part_2) + grad_G_b1 = grad_G_w1_part_1 * grad_G_w1_part_2 + + # ---- Update Gradient ---- + m5 = beta_1 * m5 + (1 - beta_1) * grad_G_w1 + v5 = beta_2 * v5 + (1 - beta_2) * grad_G_w1 ** 2 + + m6 = beta_1 * m6 + (1 - beta_1) * grad_G_b1 + v6 = beta_2 * v6 + (1 - beta_2) * grad_G_b1 ** 2 + + m7 = beta_1 * m7 + (1 - beta_1) * grad_G_w2 + v7 = beta_2 * v7 + (1 - beta_2) * grad_G_w2 ** 2 + + m8 = beta_1 * m8 + (1 - beta_1) * grad_G_b2 + v8 = beta_2 * v8 + (1 - beta_2) * grad_G_b2 ** 2 + + m9 = beta_1 * m9 + (1 - beta_1) * grad_G_w3 + v9 = beta_2 * v9 + (1 - beta_2) * grad_G_w3 ** 2 + + m10 = beta_1 * m10 + (1 - beta_1) * grad_G_b3 + v10 = beta_2 * v10 + (1 - beta_2) * grad_G_b3 ** 2 + + m11 = beta_1 * m11 + (1 - beta_1) * grad_G_w4 + v11 = beta_2 * v11 + (1 - beta_2) * grad_G_w4 ** 2 + + m12 = beta_1 * m12 + (1 - beta_1) * grad_G_b4 + v12 = beta_2 * v12 + (1 - beta_2) * grad_G_b4 ** 2 + + m13 = beta_1 * m13 + (1 - beta_1) * grad_G_w5 + v13 = beta_2 * v13 + (1 - beta_2) * grad_G_w5 ** 2 + + m14 = beta_1 * m14 + (1 - beta_1) * grad_G_b5 + v14 = beta_2 * v14 + (1 - beta_2) * grad_G_b5 ** 2 + + m15 = beta_1 * m15 + (1 - beta_1) * grad_G_w6 + v15 = beta_2 * v15 + (1 - beta_2) * grad_G_w6 ** 2 + + m16 = beta_1 * m16 + (1 - beta_1) * grad_G_b6 + v16 = beta_2 * v16 + (1 - beta_2) * grad_G_b6 ** 2 + + m17 = beta_1 * m17 + (1 - beta_1) * grad_G_w7 + v17 = beta_2 * v17 + (1 - beta_2) * grad_G_w7 ** 2 + + m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7 + v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2 + + G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2) ) + eps)) * (m5/(1-beta_1)) + G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2) ) + eps)) * (m6/(1-beta_1)) + + G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2) ) + eps)) * (m7/(1-beta_1)) + G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2) ) + eps)) * (m8/(1-beta_1)) + + G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2) ) + eps)) * (m9/(1-beta_1)) + G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2) ) + eps)) * (m10/(1-beta_1)) + + G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2) ) + eps)) * (m11/(1-beta_1)) + G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2) ) + eps)) * (m12/(1-beta_1)) + + G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2) ) + eps)) * (m13/(1-beta_1)) + G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2) ) + eps)) * (m14/(1-beta_1)) + + G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2) ) + eps)) * (m15/(1-beta_1)) + G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2) ) + eps)) * (m16/(1-beta_1)) + + G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2) ) + eps)) * (m17/(1-beta_1)) + G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2) ) + eps)) * (m18/(1-beta_1)) + + # --- Print Error ---- + #print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') + + if iter == 0: + learing_rate = learing_rate * 0.01 + if iter == 40: + learing_rate = learing_rate * 0.01 + + # ---- Print to Out put ---- + if iter%10 == 0: + + print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') + print('--------- Show Example Result See Tab Above ----------') + print('--------- Wait for the image to load ---------') + Z = np.random.uniform(-1., 1., size=[16, G_input]) + + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + fig = plot(current_fake_data) + fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+"_Ginput_"+str(G_input)+ \ + "_hiddenone"+str(hidden_input) + "_hiddentwo"+str(hidden_input2) + "_LR_" + str(learing_rate) + ), bbox_inches='tight') +#for complete explanation visit https://towardsdatascience.com/only-numpy-implementing-gan-general-adversarial-networks-and-adam-optimizer-using-numpy-with-2a7e4e032021 +# -- end code -- From f47b744cddff906be57558634f9de845ce23910d Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Fri, 25 Oct 2019 00:32:10 +0530 Subject: [PATCH 2/9] gan update --- neural_network/GAN.py | 2 +- .../MNIST_data/train-labels-idx1-ubyte.gz | Bin 0 -> 28881 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 neural_network/MNIST_data/train-labels-idx1-ubyte.gz diff --git a/neural_network/GAN.py b/neural_network/GAN.py index 634d3b40cc24..29a88e22638f 100644 --- a/neural_network/GAN.py +++ b/neural_network/GAN.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec -random_numer = int(input("Please Input a Random Number to Seed")) +random_numer = 42 np.random.seed(random_numer) diff --git a/neural_network/MNIST_data/train-labels-idx1-ubyte.gz b/neural_network/MNIST_data/train-labels-idx1-ubyte.gz new file mode 100644 index 0000000000000000000000000000000000000000..707a576bb523304d5b674de436c0779d77b7d480 GIT binary patch literal 28881 zcmV(tKX>KiSVPa)$b1rFQcrh(?VtI6B0DYYaq9nO(M7b}w z)cJi1<2^7-t9>rv$2ukcxy~K`-XFNV)-rA@ zrIohpStI^>%v*aubL>6ZnP=@jR=szX9+%T@e9gGK+;(S;A8oz$_@)uEtiESlJwMu7@9`tEthTwe_>d9zGisYXZnn(3 z<{q_=GENzNoxRH4bH~rE6Zbz`yQSZ?{p=nee&Z%aeET>zo@zX@cx>?janr5exbOHq z?lS%`R$O+xs`!92%8E4+FDD*#ysEgI!)3+=pTYv!@%H2C9RBs> zPUb!Fk?}s`LB{>=+xqS;)muIXiC4DIxqHN?mb1?CUB#{O3ww4JFRLI{cD$X=FT~x6?d^~FI-DJka)$hYI{7XSllBdGK$}bC=k1Hu-z|fDHdJaQ#_>=3#7-6-FwIV z#|6X_DmR|mW#`?9Ii07ud3v#R;-hbg7dqls#^p;^tOo9Uv5(`=v18&#c}hFiLELvd z*874+d{lg7EU8;pI)s))cB&nBXYk*aZT5m)0)#s$QCxhq}-4=C2>obmYL z8OAlNRv%w?8jTLKi#)617V)UGZ67DLtbgbXn0z zcz(oL?uj#F69bJsv9Ww(Y{v*?vD8Yefadey42pY+zl&@2Y%LjyS+bQl8BdRp9_MIm z?Reldel4CvME3acI0GB8DHeCE(Lz9oQz@Prr%HTdtiKyaLL4jceqzhdh>qtFDLC-_ zloKBiZ!Mz5i5JJ-ibxu3r5Y;YMNEP%aiFaDth0GUBYuu!Hl9~JocO3XpNXCE7&tvj zVW|w>qxT%qFdiaj(2AdnJR-g}Hh)}xLwAHgXyh1hrzR8KtHxS-1Ka8I#D zVmTBpgsW?8*N9hfRPh7};c@Npl@V0p_d3x&j!uu|h?23o8|y2Mti?GuVmHOk*^`wK zf7#;UL*hKmEXRk%7sNe99B%QMoEDKijKlTC@5Y^wF-5E(#fVF;90Q~W z9EovFapJ~qCsxKKo71-Wo44 z;$XysNlfSXB-e|BE8fB4tq{;+pBD3_jI>-Fn>8Y;Ycwu{JB}AQV_$Oa#`#MCWMA>n z;<3z;aiSX8;`%H3!C^`9*Re0-Zcgl-lS?-l{Vokc9@nVKAaKkhYFY+P}iYOyim zJK{CQmMSemV#JHaAJ|N>W(kb($jDP7C=uM_e_|a)_#z#Nzc5#f2pp%1=~k?!IPhXa z#Eryuz47v5A;ojNZxtOfEHf73LPYhrL#}=rF^{Wl z@#PU*3;^+q5uW3M>Lw<3Bk`9HE|e5!NH| zj7#BaV*y0KjYZZ8v~e!2$Tur{EmFQnX-T}|Bh9=IA%U}#(;`j}66q)#;>I~`CRfR> zj6W4mxi}m*nN`VuAMxRM9>k+1Hj<>p<;6Y5R}v-SJKc=&z9X57uZ=ewuRp>gKXu{| zjrSG57}-`FRB>O2*dC``BwmqS$H7zBiSZSYl*D=<;fgpHZzGaJ4$eX45h;7bgoud| zei~&@th(hMigg?j^)M~t;6^}NSL;PQl z?acFyODw!WJ~I9s$Ne;&IbLwQeNNLj^gVq<+rp4=+IHJ%WsZ5-|7%&2V|g?Bu$*!pn?hto1%Lm}Uu)*SKlBAQT{ zSL#)t{-OMcGclG(TzxYcB!>93$$tjHD1J1aZzS5p#t7V%zgW~jh1+V^Jcz{*t18ZB zK0mG}R#4}t8U)Qk{EnMAoE@>YBbW^m2Qt>U-MH#P>Oige~u_#c|pD%ROR0 zP@GaVuh_DMtvHCDB%+j#v3%pMBVG2e+ngcc> z^8~JmRWLdK+0>D)QStGV`JDJqJO`E`+w8^%@Sw>uqS|8T#3hk65CVv0ykx-)YWqr{ zWfNH=EGl5DyUmL=au!nCQr(w@iYdN3elPwuUXu8Yi98kS0D9_VGjYx?u#gDbM9^W{ z88N96`{SS3m9ecjcDQJvjsh%DM5*G!H&LaNND0iu%IciYNJeB=g)|jRAoiO%3y(EI$n@-ugyN8l znWx6f0a}buHcdbxNEXr?iq}S>65A~jC{w_AuI$!$RIyvZg*G4%B^(PS-tcD|MUo z1b-J1m_wffFs_{)c3fdrLlHZHPN)DaE;XjS$Z} z1p|se20%7E?u{TsCQ98{f|tZTj($K_LE(t=lS+-T{Dg=@+Aro*HW#bc zGs4x&T^}>bB_`hz&5qCRvZA$Rg<$eIT&01}eW73y16_?h!~V zb{FU`>mlwh%HGaHiW>oB;IU6qtP8G7w%B-%@zBWUV;L^PYeY$Igk@U*-3btJw_;aQ zEF-b2YXwtYZAtuC)|*3oI#Qx(p-TlGVUfcoD#1v^8ns#Fz@5Y)U`(*&*iG>|jvqcv z2QN8qBm0O0pW}W|M8~CoZ*cllc7EKNXT~D_a5q4)Kr@syu^Hp*IB;l!@P;WG;)=*! z$<+X(DCln1?OP0EP?H+Nw{L8`{9~GvPf`oq2)AnhEIy>k*4AgdL> z+JFjT1B2a8woMlgsBHOIZt$fZ{_$XZCo`31LXv)15f=9e3V6xI2m zKG~d^KuuKIK&YNb5rN4f4k$>J>#X6(JIFT43@-;+{Gqbki-2!2JpuKl%$u7^sfYq3 z+FQ)uubB)%oTwd96`T@y9Xted3cw`ZC4dWPz!D+3X1=jbSxd1VY@WorBaw~A5>a|` zPSUSw)7BRGd;Dd*)<#xHjOH!y9$IxjU%O(QM&s2P%V;G&XFYL@y|nk{wc|voG`p|s zk|OqI>Olz={}n$d+DNe6ps?gxrkUFa0I&owK^kNRL&X3LLk#3hjs$&Bxow|8Jjqyd z6d8rgnpIw`lt6w+Qtgnm*X4_?BP!XfZkwpPS>vG^B#zSJjK{$t3xog;8Cy3#g1$!2 zw@LYTDb`}c$K7##LyUoDL!|JrelImjtb$EUr~4CoF7^s|SS4?1a83iqx58k2e^B2Slh%&KM2>Vk)hUShS7aC%{D9r1xeBGaen|QF#%XjLpqT z!D+?@1S1s3>B}=9yvDUgd?WK1JkLU^5f`>VV#v!r-GP&*F#7$6E z8YKugybFKvhf@s`2+2Sr^a z&H}||nKl4FuK8@ME_FLMcli-?^ z8(Eb;RK2@*x@#zY()|3`OJ^xk{YTs%WYDA-9A32kPWI?1p?%KO(@oX@=`$?_3KW4C z*&(cj4(ds_f(wWRAF+u#2+Rbq{Vv%MgY)WumNnn%5-26YvPt5zScY~Vi>FKSRhCe2 ztnr8vM?OHAC-WKBy3B&%&Y>Y3tF@Cs40h zVpGTBjJ34cF<^T($_V5>2$ECkuWr+~+ntJUSU?nFs0mSgt!AXHqIbjK zOL7iYR}(n`i;L~IpbiXz(t)x-{tLMOsFqbK|Bxu380>MVC={E{FPaML=iBdtsw`7*X*>PBo{U`v{^`m7Mlg7!iN0O*jGw4rF75-MD+sO3n>; zS*0Q`)Rl#NgBEcELIs@%7nzjdws5WM8k@U|=Ewb-L%KX^GrT?%TMx{(PoJi?Fn?bPRUy*(TX>u}txR^q*g}DC9Zu(MzNuzK&xp?hPmdsJKv~Ud}ZLGfh|=!iunH z0w`q9ge1lX_;t{*m}@z7#2MqR-B0~*>>HIaxsb#1+q_OCmyDSS!lAhhzxfny9MZL z(w+l$J+vE1vjOA?JhASh$XlE=ou(Mnu_%&3a3XJrlMCk0<$UQt#)BM!G!uqgquOiG zFDU#7e5?u@64T~37Xg?TYmvYmK{yuduq-V6>a^WoIA}LWId^ZXP8e2h9Mv{U&fP?W zyLOlCB)FU)9xC?E&G80A&Wvb;zzui$~p}oh(l;|9z__WPWEvF_4H=v3viKud+jEz9|n-s zUvqE36&oD50F)2IG^-KkVhdMtdWoQrjFN*#F-MU)*tQXl(7Oay4BepHsMLZmm9(qeBsg8|LBlR9aEhHTqc10{N zAU88>z@@myF0Fz36x^5!7_wrQvlwBP@DlHutZK;2G!Jg#U)P4o?TOQp_Z;i5ZWTk3 z%S$Fv5erQ8jYhmlU?o;nc1WaMG}Jns71nkoxWJQw@pK5Qpv`gM`8+PmjMXm*DRvUH z36~r!j5zMOh)cfOXu%CQl#@3z>CCZnXc4IrECHSrHBgmgOLq`lh0KQ6LBxs3POV;r|EF*Ha;k3mTycoU=aHq6 zFn4$coT!v}?4~9S`P#CIXh2lp#Dd8O-?Y0b$amS94d}80DRd~Wkcq2nG4@-8ym%S# z0;rFT{?`+qSk)dHcFbbpY08_v)?&QZC4n|pU(0?TJtWZa4PlxR(Ry)mzNUf73eIbbTODvG=iG5 zCs;1T>U>@>3<1UDcU|)yTER8Zv~$f()^NdbY{!;>RZyaIfV8| z$Ql9H%*guVTnzJO1JJt@P9fe!x7)K{#}ULk9vyKvjv12reoSy}{6bTY%ZbQHZ-oNN z_s5AtJB}Kr!8nWO!bNfMgcf?^^rj090DFRJExJ` zL^wURH^G8Nalgb{zvf@DnPY{J`gYNvVPrA26x~V=QBO+j>#dmlxKcBU7g{PJ7A-@H z488+cM~UB|@N`INaWi1tMc5mdvQtUH_%`X@ENz&!#r7`SDc*6DB{9iZrTpJqWIo(T>!jM>W~)zjlog{vIKVIV(FW7z``Q+IZ_9>_PjX?aNeA}%0>hwibWXY zG^YZkzXH+`k(&FUNN)gW@t+A*IzkM94H#S{E3Xn2VA)S&DP6Qnc-DTZy(Jr+qF-eXvWM#0VPN4>Jwdw1SBoMn`k!1xh z3sz3s@$A@k1`G5IyMVS>RA&^tHmxV=pG+&s?_P)9Y*=k7Cji2!J!J>MGywrFFwG0!QIsMe zS+Lt4-@Vy_uw0vVlilvqoO*YI3qDKWK!|Ipe+{OYlNL)gXUc)W8ZXyo^SZl)i(p>+NdF_2S6a#tS1w+@Zk5p$wVM61Om*IqN>oNThJ-9sb_#f z;2Mzl#zI^a9r17g8SHdHl)Su|$u^I(GI9k_uVcU^h@y|8PYDr%HNcCY*rOZIWm*vo zc{InI4E*-vNAU66(=8)?5fwdB7~~sC%0J0^GZh0~0A__P<*^*gbdkq%K@_%}5OZeo z34cPN5*1cNQ}iscv>~cP3S%8@e$@jS`2*TRB}=e6e6CC3j%iHs8T2j|#tMT|I8v4K zYU5)Qn?^9@+Od+`fTCg*w6Xzzd0G6Bjca;S6oD{LX$?`Q(xy-^M4L6T52%N#CspLA z*CD75RtS_)4*Cn*4Ymht+eXC$Zx#N>vZ)n;`gSR3RG%DB>Lz11K@O+eP*oDyZR_s; z3Ts1Ju#F4fVfps^PycY9wk1NOH7Q3RBnw*1^lrrgsyRCCt4S|m5t557C{kYP37GZ7 z;3F%ChIJCWy3ah4LGo-w4k!}igMc{tVy$hu2c*F)d5%f$djJ~{@hB|GG);#)NIp;+ z$rKVeiE+o+QB+ATyJ@Lp$}VEFs?Gk^6!Rl2NAX5}Ln;TzU2cMU3z0{>9&%PFY?C5} zwMp4J?RcjsktlGvfFA3Pc0ha;93V?Ys+11J48kXUAnKCKX*shyu&ldb3slSP_LL~| z)#g_nM^UFE*#g;zkA{+mV=N*OVTA5p9JW_>v2X$;P=jc-YsZlVdL2Ikr&Kwu4NP$A zscdW}TU+Xew?xSoPZOdasZLKW2mEt#0jRd!?)$O^OFqH&nRf*wsd!KoAZ}={6M{HQ z{SE0Z9t6~6b6FZK6R^M10V41rxdX=H6t3vHic%U5T=gY%NFp{ug&>}eW}n-B9XKZ` z0N4wPCD{*KeH*{rZ?+Ji19HltMw7JoSTs#EygcNXmt{{=jyAbp>N%zdKj%wJx&xn$ zsuqDB+CF+rWPpSbB1ilU@B&pTu@$&zQml}9LkT+Ik&u@V)}Y$rR*NL@DNs8+Pyy#y zEd{tqZVV@T5zU?=+~x?bcKhgJc>AFqbaGkE$ot@fi|kXRMW7U4LRLm|+~^3eu&MX5 zRX>q*!V!_ugNiQxsE&&5Xg5-g>yQjYlqbg`FMvOS!jI0Qdc6zJ_j`}&?7NQjtM*`$0i6fu}&~fNe(J_v!no4j|X%lVXV0Xwo zkY)(A@UhGNG|=6CC;_mj)Tnmf&QtEMn{f|yCA8g^nFY*Ko&ec?$y)Bx9U<>QdoI-- z{PRGgL@GUnO3)!tI84$hx&^!|!VxQ&JbI_$K+-kA0z54%G0m^J&i?*kkt;!Qb*eDx6z@V^43i6jEAYvyEH0LlT#gg?nWqw;@)LX z;v;~T0LC@x1Ib5Idl0XKBAVKSr%sIMC$g7*wi4O2c&5C#@zGSK=qopXJ1%4eI+o{%+B*q(yoeTK` z{{j#4T~oHq1)k#?Mh8!mSyeoLD$sDruA##h;yBE5d2pVY*PN`|e>v7f$69pTkB~8x zl(-3`NDB~YA7KaBW%YDMU~O+NQM3~_5X-lbz>xkm)OHBS(PU2VT|f}pbz$o(n&0kL zHfDba-ISdq$4@pIF?Z=J5-Sm6sAxIN&d4MIF6i)WBjAFpID8@!zG(SQVmovb7A-X- z&we1UJTsdGV5yzpwZx~v$d?qg*%j!cNo@-01>K;Z60nd12x7!#HF5yx;E+v4->%%j z>z=Uuv6Ycb0AMnnQ4!2Ru?&BYg~(9?F@Tc~rr+ivciu8Als{Mt03h1n zjxRu<`T0Aq;5{6+C7+$SKC3QOyPkW!y4K*r`u(?HpFbSq1~%>dwu zuv$uRG=k)Ad(beE<23)yh!^x(61UtpgR|N8aYvw1>e2fMM6LY-L#c}s(^t9o#Jlu4|NIMmnIL9 zcA>D=fFjyB%Rp?eoMqN8OOCenkxOGeAmW1CLbVp?k7av|eJ@6vm(`~@O7<}XW>27& z;0{|6ok=B1(6uH3LsQhXcc?@D3~Z2QeM_7GFd~he_n}~xdhwu8s6v=q`Zf*H9yTt^ zs>oheE=-?!0K!7C-LffTHDaRyiRgfuu&u6H4R%cS!R?`41LcyBTAp^X+k=@ZGC<3G z7uKOV^dx-Htb>5bAX1*nYr!Ce{`6$0(b3oOLzYS92o=-rGT34GP>haXP@!1^v_RR4v7N94$UfY)^L!+T z;W9;eN7{Ng8ECalhzZDD=<-mN!}cT1L8l3!cMDT&<9{zf#8Zx)m0G$9qgvq#Q3verp9s*^r@NA1Maa>AZSDMR{MJGGEyA@TFau-@F?PO zL+rpZK>UuPl;%sPzUz>5jh|r8(gW!;5=G-QIY`y$$Md(wQoLc>wTET4Tqoj}Iz(6Sy8lewLrw;EE*--$YQZU#I(2Vs1Ks__H#gLDP ze+KPvRH*bbXrgZ&8{iyOG^!X!_}_qt)a)Wpm|YxqrAcI)x@}of0$Lq5VWestT!?=t zU+C%q^`!$8B)BC%GrNjC4Wz(-JQ;^;{x4~wy`yvKo3f{heaEgPPt8VzdVstzk%9T z^frPB5Wy%sK|r@B{{11b;7T)9l%VIbFtVpc!kSTdO9a4!4y92OWW!UXjnwcpTMImS zDzkLE5E-8`WZBlJy*V+dz&WnLV}WwmvGTqf`S`#HYTF-$j6oGc8@?$bz0x+#KnH1sQ~HH88ZAfYned(@mi@b(2`hVGPA4 z9`_{EK`EY*ohn*-3uFhB=da&%RS|d+nt5x3;z_y1p_|~j04bD^cF#Tq~!ueC)@~J9QfZmRk>2|;ipa) z#Zu32bGh5uqc2*N6~%k!}Y%JpGVvXbntb@Nqa)bixi=psZ@9CWiE0P6s# zm|9WON&52~%geKbSis-L6P-Bx&_EEBgX&yVdWt|=H7`ZKO|>!kS8G}tfb^&~Z8AV@ zY*4IqOnA(Isp?5MDOb2G4rJJm0~beID+Il*CJwlSWU-}%K=!;nQRt#tCCH*G>gbz1 z==3){uLE0>;mUiDUxxFfJaf_?#MXdwp+D{e=_u(VD&#>i+;Q9>fFGm{6Y9;TKgm{K z81Mm|*@f>ZextUJn1@pUrbOR|@ z`$yXfg`L>lv8fN*>VfP``FH1aYR>a2!1>LrX{pXHGCJv0#MRLsgfUHaOY4!h4?;zc z+5fh|j+14C4X%!ZJiemSY9#i-h9I9?$tjB8pqrvHRK7ReT2>mblvRAui%9o@Bx^=Z zZiw*rCb``Vp;}p*u`4!{^X_}8u1p0%!MzBKHl zCz-AGm%W|m$%|tsP65l)UxB&24AM6^OGA+y@;)TP@&M`hp?ZUtrq|4R25Y11|AlC` zklG^K!1hRF8At$`2|XFzYpPZPgQmV&6eYvRISN0Cc5hmIW^x!!&;M|bD!7NRXilj0`ADegvcpveBS6hLuC2dPeyM>c(HbBAl6t?Ap9zOK+Bfzp_K^VcPs$MBWKG&8F(}$=`vGajT2$VV9t?=PG zIu37{5!L=0xm}_I>vN9=@_+6|EH;@EoaaS)7zMC6FxkhEG?GvO+YC$%fD6f4JajZs zR@dMwuuu`ik%6KoY|=alg0RXs2Zo|yC_B^D3D+Y9&se~aI5jhv^m?d|9i@RF2|8@~ z)C}-C^eLIC3)QX%ktM5ci1|x(HEJF>^D@QM|KaieA-zQARZGlJI+K04@`1?TJ!bP?2E4FDw7D@ zRnOO-UuzLNovu^%X=dP)jtVUnRO|YFB5RhWM>&eSfHsAP;|*?Qe0)cMCkujFa(b+y zt>X$R^Um#j%dae^=?@+ zJ&vr8I#s$>!p&@L=*?6t#dv8b{m2MA6cIH_LYKg(xQdhiQer`Ap(ZQdBsQhX0*l*~ zmjG~#Ck6xtat9LQy5zqSN{&9xf>(5S0fjkh(;s1;OhAWxyXl96bHW>ezSQ*EqZFgs zfesEO2{rb5=Q-yzejVre(;My0OvdvpU1n0l_(P0fjj4gr!d;(Bt#PGS&{>5Xh}3)1 zu6W%~U;^CTv+16gE{_c(Gl&>k6WPD2(U&T{Vz*)0QU?P^xV~{gL&IJT`3RdZDx~KHOvCvA-p%<; z&lJnZWSWxHgYR$c^X1K<(9npD#);48l8?!GA9e)v)?p`Aja1eprvF&> zU(%BxiELfRL8RI?V|htRK}%2Ih2jBG72wO%pn$4o!EOQT1HM_lE3L?Iwj~|n2v5q@ zk!eeJA45r&pMLVN$WV9DOKVH#y5SrG5i2^+B%fIF5Doy-n%3R=q5wY@ig6KqD!s0* zygh~j9L8kKWcH3TY?Ylbvs;gdMA5xs8}(clXJra@Fb5n8B?(koQj0{lk$}_s9@zH4 zjno#DumX|ib{W@nhQa4(en@bfG_iWxCu#*ujVtouF0RH~4wPE*H6Qg8{bl&Wh*3r| zE3&_q@v@4IlZiJsHhvc{z%e7(%6iNRs-XfpAve0PRiqbb3mAvO1$aqQWU681=TaF( z^>UMqr9AxVzXF&}3mp^Vt$Yi6b4kEFu+xXytNUK4-rMY-SCO7mtkG?y za?G7MT~@1uQM3@YN>TYSez9!rt5JO0PMlasSOH);S*N5oKz5lXq9T=$lS3T&d@=Q<|)Y`#cxR%eM3EoUaQoWvkId+Abz`sV;ig#ts!N zB@n&2?!*Z0Yv}@sUV=l-GlVzaQV2;)0t0@w{>;t;jVUtb!9)`{zltytUqEXdJ6+IH22Cd*wF=MEf z+CoYgW*|V`He&%xA^%DlUF-wNY9@xflI&J4B_mRcDWsITUjP&=LutC|UJZ`fVDjo! zL9YwyD^)H``nV-EqeF(S`seWww)=XD<(-o~t)7bmAyg8=k!+9}(-|dC@_dywJuk$*}IMM=KuGvc+a%DRFNfGA%twdO*@_*PpIGL`yRZ!jaMdzI~V4G2)c111)?y5}C5 zfp51I&`Ytk7Y}0I|u<5sY)GILt+a*#xWKqa}=#y}q zT2Y_@a9jc?Iy(;ngM@%OcQaPB8|Zm3Dg{^mQgR<2rz#ikGqmiP!q~pzffu;vj{kbu{hWT8#olR5|t~iFl%R^1T00XYa@C zJ$4xgDdeHDW+l_uHA}Ii9WibRgn&eML%zR;7nX<;b$~5AmaH;BWMPl#gDol|pV1LG zCTr8y39g{(dz(PE*o{u@Q~X66Sj+)@C4|flG9bGf6^EwAWgc8nD1 zzg|^^UgQ1I=clgZSa-e4X+|j=qOY7?E!+zw$k!~}r(EAON?+! z%9tb{fuvp}ei)f(UfE#Al%B?!SQK-TrQOn;mZt1avWao`t9;0nlk)|!aj=$Yn~A^b zq-Az_>?s{lmgJL{=QBK>>*29dwgm;pmNWWV!IR+hv~=7_Fd=7blE%AjKZ2efGJH6A zy_F=P!p=MR5u!(oj*uKKEl`xy<57UoQBzwnv2(j-C*vt|rGFVG$VBIl(*-K2qt$(R z3>yqnjZWwrPPf4T!j}QSOcF1GWWEQ`zvXbMH@N_ITXh2}BbGZoSkx#GB&Y&g=2}a7 zg0_3fLV65J+w4*ytGQFx*SS0`e9jr##l|1Rp{gwt4JmT3v;kg^6Vh~SI5XwoPELct zv&D&>Awv;K-_F*OAEuijpN`1y=nMn-m?x8&-X@1O~|WWSGJ4Gs#M&_-R@#<_ss}U?b3`VIvB$NMB0U-A ze4DOGv#UJpB~2|;mc2{jpx*SL5AdGnt9+JxI;ZsgCZpUpS0kN&k}+6)GL8s=O2iY} zEEztYu1b9QC|cUn?&mxPXGgN6U&~PkrB;nHJ;mM#PBE35M^p@>IY2t?d6#bocn4^c z{tA&7E-97v(uV|pm}{b;r*!3*Lo_&J@pp*@%#EO7)4(Aqi1Y@3{6~Jgj1#t-R;&y-v!=rs$ zX~O}a-5U3u;NEGWSM z3oo|IDDT07kqyLOk?hO8Ha8&B(Ye7fV(df;QBxlEg|t3uf}fHDnpS*`x}X|dV76~f zjayoP(myqVTMY6xA-aOjrQ?|=nOlt_US4U}2QqJzJ+HG4u#%P1Ob zy=_$vd2T2Q<&qxP+%l>_qJuICwPbEEX{wY>Z5(hm#+Znc1UzfxJOXZxhj`Uu z0M#~a5LByx48l-uQRqxL9Orus@ zuRAnc`#>HBP~k|Q8A~_}Tr#wsC1-{V3ur;5!{Tqq%_6dB0iv{!HdjdrZy#CPKESCT~u@VZ__@ZkI#UKvg+#nNGkWiN)mfLM4jq z4AsdfEOn$NJ>;NEy%23Q(z7NjU(e@T(zsQNLbf9MRLo4Z zi{s5sx_LypMvuAGu@ihXOK~;>1s2)Y)MgN|&zo<#jLvGJFg(UikrE3=8SEncy#R9= zB(ZG*lbr)rApfq~0qseW76?U;!QmtSIS*I$*6)G=49xUFg-QN$9hpgj%1#j;#IY0U7{efV3sn?H=8CDfwXn!7 zNDcHN{KHU{DGsA~=YFJv#l`bsJPi$j)rh1I&~QqPXPZk}MQwNbL;BB9+cw>@AmM5H z0}T;4QZIKcfSt00x(82+mU|408`2x%bUDXVYOFj{Emkm^+yQwT?+Fol#A{vA2g+j@ zsE`_uDe7}lq8EUfIuh{I-A}Pb9cS$RB|{@U-&I>Tl@B1SjNzjnSv`KKDD6DbjX#cC zZH`AEgI|QHu7f>97*uwP*&L2lxtcz~q@eU$Ntui^6hSP!E!beRmUnxZ?Rj%-#&eZ( zYQmUSl;h{$j^;`bW%<#js8`5@ zVccS5c1o#7Q^G_Angk5WyaVI2YFmw?uCru;TFA1x^!MqAvZrvO~$oWx}?k5}4OmuDzTS zeyh{oIrGVHHP(U;1p&sAN!14vVvg^Bi=iM`n@`0z#GPfaK$QcDmB82>C#I+sMF~{~ zi;W@)9Yz;4hJ-L=RkG!a$l-80D$}W|Hk<pg0``Fx;><4SKXeawZAaeM&(wYi~JH zig!MdnL?49wAbdBbFX(7_Cl?>yZP>tZ?k?r7~289FhS3HH63oSLU%%?GG!rsB`;M^ zGVfPnsJ#r=kVqsW`kd)Y@d9&l<8Mp}4Mspc%E_FB7qVAKb!4n$aJWMu&CPYH@({wFF$qmH+QVqiFD0t zDM0)nx2?8u6#gLBNz*|%aPr#ey+TT8PV22U`=-Bw{*(@Nu?WFx5I+EMN|i`W%#aZE z@iDuzC|eR)m_5~y4$!o=o!+uiNpzZqkBjuQll8o4{tZa9sLeVW5eIEaurlVHQ_1$<)b7N_ATBm!1IAIGg} zjqH@JnF4d|!E|~%%KEhDC^kj;&ca^jb=Zpr5B4s>DH=Y+M{SHIl(M6GW;<_I#nUF~ zfto%4@g^is4G*iixjr=%(adpkrmq8kvO&+3RfD_>8)r2S!H3B>Hv3iFSIOIRZ?RfcO>Ea8 znH@$hI}^yu7*>gRie7#3MZ(NU+Y$P4Rm_N}j6?cX+$Kgr)zqbNg0g6`gq${iYsu6{v}PVt`x4_=QuE&xw6$& z6sX8skf6Z+MKS|5aS(ej2gD<9>pbqX9t?4p2;6uB;B^HkTHWDC-Rwvb$J9eJm>XuM z;}mFS#e3}8kvN&$Ga8&cZE5@JKsat5z{@r~85Y%(bq85(X}U~Io#ARa2fgVR-T3OD z*5-_c~H&jJMv|G2CFMN3*I23}CP zr~YZQ{g1*_zzPt6;jE?WczDWAHBB5|RIB8M)94@=Mv{!~eA+&TCL|PMkZ)E(YB)-g zLh94-r?+CdUut$@ZJ>qxPQe+H?0eMkdQMUka%!p%Nji>GT}c9#tSV3(7BFx{IuI$Q zC=TB3S}AihIb@D{%65!`$a} z$1WtY30+JXIfHj;Q>CKj`KZjn3q|hs(8nQ>rS|xh8*P(nh1#@=rXJotcbk29EIPc2QxAx}=ipH* zTo^*+WS!gKTm>XS#P{ZBb$zMQ+**gD$5TrW!S>ad0}J3^uP? zT5xos|19}{TP3&YpxG6*0o+riT#2tvX{I;=Z9dueTE-H+2x(n(to$7HWa}xVC63>K zXjPeIp2~T+$5J1rBMSBR;*x87Ec0!>x9u+2JbDv@M?L{(BtFmMm#gTyO`&LwJ(1)e zhi0ltva88C&5#L@EKE)8c8GzH0b^gu+XnJe2WYpbvkOQRg&?}4TyS+shgAIaVLVYy zGwRfQs@pSNA|W~4TSFyTjv33>)3Q`qz{=9`j77J{EG%S6di9YCAyUScL?)?(Pwh(~ z>)?RsC|LdHEC*l%nSK#Lz>fCR;p#eA7}JiW6)5~Q%5a&RU0W1ZgghkGo|Faba8L!& z(yYR(D=x9hRnuf^y(ug@O3R)ZKeuu4q=Q(0V*^y^Gcd(SLp-oq=9^h_+fjwl4^?SR zPmH3ndWqs#^mi>a(m_vLiLI~XQg`F*Y|vR?_KV(MToc)Lofe`# zAGnx1aFn8)dJECA{hW6?j6HA=?kh;oGZc5asr zJ5azJ{sSQj8z^8w2QRbC03fej{U)Q!VeB<~%k*^G^X%$Rk$-eOu_{RB4BbUDPb*{i zrH%F^!`c%6@47as4ayPAP$PBq8&0|a%7f`oN8u19iO%30!)bnKp?JXLfUsYyHi9t1 z$*nk@7$2eUClm_MKa}gJ-x_C@|3%g342Z`}FXmmtobrgH4ZEi45#{}0o4`mUhrTe? z8pr@rfJQA0#MR(;+H~M-aH0^TFeGRI2#Bki(*aW{(#`cqR_IMYPj`#~-C3H*U8`Sg zJ~6}7f<&z~R4(zGuC`Vwm);^wLr;gj><+yhDIO;KV7?hp_aTb?54G~KAB$H8OdDGM zO!9Fdz@4HD!k^*ISX9hHr_$-z$PPH7rem?EB5GK!^Z^;JiDNch1b!D$7l}94OJKGR z%CVacL)#hD4gl$raGWX!waZHCI&?f8gqXZQq&&|LemiU0A!|C--L&bzi9~^(qJif$d{o3V@dbJlGKQdDb>ozxd77!0-k{`ilK$#66;_%P+29x<(pWh< zK{LC;=`|=oyM*2AIE>-E4Dvx>+mntQvTAWdBH>pKbg$+Ju)?Z3Q&NAK^6Pn_HbKsk z$Lu@5ySc|w2Y6M-p%)~nbPyU6ZL2owDe_4vGF{aBG~E@;b2-9$->SVJzK7U{+JTl? zNuoUemIg8?A2e=c>6ls2uu`Dy@Ls}n7c)mLirS2sGe{w}e%xv}(hp@j&9LmD-m$AOuXP9h^+1I-_3d`NG@r3;rQ&P>L|Pe=Sh z=5e?lI4uAY6*HLtp#}aSoC7_^)vMLy;|#T?>5DG;;?0hKE&#skq1XMK@PH<&X;lq+fmF5$0I)I0$YfkE3c64Db(|1O5j*I(dhQI2Jjm*!aq| z#lf#TmjEZVnF%!ci{_x{L+MITaH+1P8&dQKc8}}zWG*#KwLN9D{zjU$405`RQ|Tht z$BYdOqzZPh$DuY%ByiB!ZjMqY`c5ndT!~qgpTh3W)W7D$?(rNqmT1v=E<%D|D>1UP zZ)vIk1*V3kk2Jrqmq4w?Q0}3yh;dI%F9q4`Bq7LP!uHJK54(36p?tRRIA5nG~f?&++as zwvdK7vQ=PBd~?hB#qXmIJ$ey29L-d_TvtFxJUN?#>joiU&a|na(@KsF{|SML-B)-5 z$5Ejm@f_|)+ESGZJ*{z45t7K>c>YGo$KT1c@>I_|cE~tFTcQIe{?_o+VQ#dHtLjK{ zHVy76(HU5PObSJfu2e)92>fJMSFWkk8v%Fdfa~}x1Dcl)5Q@61P|-Oe$o5y(8rMlA zG&hG1QQOfbpz^RP@gtFT#i>8EnqN{#9)g*y^>qJhV(;mnUY`E?T8r!J?U~cH$pIRj zZx)0CpQEuMY(KJunL2s8A=ODmNT#Vnp#ybE3;r^#O?c(xk@Tv!c2Ibl#tS+onmHzj z-cqu@IYwze2nbwr5Gpt)ryS>fOQlk&d+i~dRMf~xEh?9C=9ED3m&@2&;AKkc0Ez=1 z(TM%Hg0?2sG|>riQ$9cKI&&R~^~SR4AWdjQx~b83(|QaeOGX{Vt0E~TPoZH3qZR=n zEGdnk)2Y$GO0Ij^(7`B~Zq$xYG~Kuo4N(eUGZ z=IY+g!||}E@5_#0fD0fv#`IO*%N1>-K4)#%eHdn5@L=u4;JZa6^W!yTZ!|uqo_NGu zNcND?v34S_tBEO83sPtHZtXl}+iUp+8U@@$YgrubXdT{=&hz%@ZOoh#^C~2n6wgD( zIdwAsuAG^i`~jdem3JV_K|1mxzaUioQ$ea zrdL(4h0gjAsYgz9!q{BYmY0T6g5zA$%+}es>+yNg7bg82b&+yr`;p<7E7?NJ!KY-F z+6Le@?jW$HFoy1ab$T}q2w<4)OlHl6hmz1^IxY2!vYz9<&?aFn7o{f>NBoNM=^~|G zs#C5a8030*dPw=kr2;dRPnqLK5Ews!O@^6n@Y(kk zk?-j(C6ZXu;qrCo%~D&6RK+QI_vs6qV_(yig?OxMGqobi5H##!y#$I|MXig2a=_!Z zY>x}S56-`nPuVE;BbiTtxf&FaB{R!J5QZFsGNhjomm3^ieatkGrmE3x7ORKy^=Q@h z)^CV9H)^NI$t6RAHe5F>K{;PBtbzW9%NI!Bw6NshZVV`7j-;YNKO|F{>O2C^Mg~PP zuz@i2JiO8pnB(4h=RAjk zZ@wi%0f-DawJdpz>-|hoL99>Z*29u|Wb3V7b#NpWRqVDjmo159S3`R-g_K?kdFon> zJ0@#sT;y08I82Hi7!Rq#mFB4+OdT&eNUG{XRCK!cJdG0FL6l`JdOYuy9_$s<52ZFRhJEWBb z4TpqoOqfn73n~r{cRWpXzR*nC{lp8BI%eynhnqRX0am1*kkZM^pwE=?$pQM;U7`q(F7h2p;0_WS;QEcB|*)3c-4XNDwwfeh5-wQYC8!4v;xEl zBpuGnLTakNu7gFkc?Z3^gv-Yd|GLkgs5&(ruk4+(o^{_-`$IL)jlzh6gjpQf4HDdZ z@_sqZO3+yV#_612c;qPVatHON8A#8TV`Y{lT`J+aQ~nY`e@=8FxQO6vbqz}l)eVva zm^nqxL#Gk`8r0d2PXkA)g872hugJ-x767ic#zvc@o9D1G%I3)7$JZ%FLbKri7$^b` zB9q`4phNjQ{->3GGUMl)@jHB$-=x&+tTlVRIK%KoI5;?=M*Fl3FzT^opCg=gKwQ#e z7?w%ptVi0_oTWMQovDV-v8F~Y`p;U4A%7=rZSvAEGa0f2r?xO0w4|XvL*`DA^1I}> z%(VS_vv$yGO@JY1NCazwsPN8yH3j3dGI2B)xAW(7Bi92o@+ zY$u>+pf>y`g^O=f4kZ&9>U54kt5C5Zvq72!#?RJ)Db$|MXN9>opRty3LM0{u*m3WA~|ms zu=_)Y6E8W3LC7G%GiL z0p|9|7CLsjw7jlWFY%;`-rSd&4krP0JqXaq=}jg;4w0LzaA4`~yxyWP18YqnK5e8K zF}13sZpok2Zj<%cOs>K4v97f6cX7>}sl)^%lHZj>uG1v5ORA=4le19DAI#ipAFs(^ z$vLA;7OHlZ+H?IIsSTD^#vH787o@{5I~lHnldk=CUWdaQDf#HT;Y?eV&dBDy>f}L! zfnl)|&+!;?95M0&ch~8U?ttDIXc4d&frp2-O%gYH0Vi5)H3!(El0sbdAnA4ARp!Fc zwI`jv{>~kD49w7RdCMY6_wDDD`rOKvzVYcsW|JPBE9Vy;o45_wSAjlYFI;ITsBP+U z;4n??wXp%A5P*0^RY8HqtQYYDj#aY6D@A1JW(HRjF(j3hc0LTdpaP=HX$~VMfy#Rd zL@-m2lL^3!Ss$dZdd`Tke%+<%={sIuke-M6ulUb)?j<=)d0ejfFyg71H0x6v9Odh@ z0-DF}(u2EX?@Y&>N}$dz6nZWO!s(cE*&XMQ};Q z!cE60GQNp}h%!!kv{j9a9fh7AzQ%dS2a?lBhQ%`!c#@71FchFGEdvhtEo#b!Ssn)( zu+R;S&;q36>YJWs>HJ*8lZyQ8_PAT~d68aOG5bwegU7})10@0>tTHy)mIK!dB0O9S zi`%Qs7Vi!IAN(fN(>RH3&32${F>s4a9^0dIStd8uuw$x7AT3O_Cm{TD*c+{6PB=b6OrA0! zqand{r9#?Mcso~;x_CXr9`kZEcO+PjI=F%u6g!msbR?*L@m*|o_YKiKnV(03zVk*; z(-8@8=u5Ui3cArH$0JHv7|nbLS=l?gvKgBG@!RK@^9TD%X7WgImNjK%O!Ha?Kxw~| z9D;(lsfM^t*q{~=OujFoDT6Yik&SkbFaAapzDQJM{Kas(1&gNG5P`GCY5-q`cLX1E zDpkYIsluAq>H{MjJ)#+uIdpd-HakxZk`y~!R^#nGXRE2nmqKDM5h?*i&aqC^MHcrv zmtA&4xwx9<()1^an~mc#oWrRIMN@U3Od&K5NEs}* zZByCHt~t)FJinL6v%62Zx}IZr@&>EU1V;|8egCOx43jfurxGV&;md|+iTTKqi)-K7 ziBVmZv8E6N`sGYCVw^<lBk z6a!0Nl$JQ?lOxn*x2BSDKfk4Ftw)PHx$()cy4Hf^XUNuM&9V1U#9bXxl*dAkqia=* zBR?RrQ%b@&YjQnFk8~FE@j0~VDzIWb%Z~>v+}W>!gYN`hB=tqG%~%ZAkn>Hwp*Tx^NjoC;NQU_&fub0(df_0oYrF?3<2A4kiCqIP6b2_*;3 zF&t7B%+|n%7|0X@d<`6=Q63|$zj=M41*#veSgkaXoVy6FV#B^W*a`h0UO70}vIp7d zj^87z;D9=ruiK$EarBu1b6x9c80i^2WMb0?+`r;;AEmd>l^#RfmrJK1G@w)R={SZr4fwNoT%Ej}N%RD@Zu-b%Q z50PPF7H6KOkAXhf;%mv^EA{}p&h1KVnBiTWJSCw3b_OjWvLlDB12F@e4u#*`BbZ| zE?CM-^G5{LJ533A=SMq2vP#bXqCPK;Uo#s^#HWb;ezVA@h)>RksgGlRWlJSGg2%T) zD*BwZX}TParj3$jFly8WlPi!;;4%S`0htE4EsQYeal9813>{fCd|nP1q|*;yP==`d zMydz2&wMO1MKN3D3A5rb0w&8M_6~0anp!bE9VMeummx$g&78UUCIgMM%RD)`BRe_H z?wqL!_9Z{}ijuBfzav{)BHr~?o=;%g6aC}P_7tNn8`XI*+z zHaqf;DbqOFSzE=FG7@rqYA1lsgKzM<1cG!@DHMV&BEB7i!6~%JRGeVa?dJC!o4&8Q z4iYZrx7ll;!ZTLk5=I3;%7MxD;DD$(9xMHbmw4+?aWIryVYIj8rr;=*-M3V)IKJoc zDQQ05-y3YNW<}J{d>SykstRwr+Ay!xBS*M%t%2rhJ22#BRJFH=U|i~ zj|7WuQ`b@jbGhqw*2&p~2MvAqcq46TIj4nCI(5xMEL6>?iffdv0Rr?|^IHu>~T7bLdxxyhFz0k@<<0Fn)*f!8w*78ZhZ;n4WCK z3Wx?Arwu5lA1S$&Q)C=IhvxKZtA^}MQx;AfoPnHkLx^{HfWs!6ECM_hod7pY^kC#v z$CIg`+)pK!^OUQMpXC{tnvQXDmQuPv_BXeNj%^OHH|ZzB%9E#)%Ac;|$DHh|-w>5+ zv0YB=969Kerad+lI03T+iQO17f`L#xA%bv5Kz4b~(%e$d>?C8E1R7o1v7kUNej@!0)&hSC z{fX=!>gLiUjU~8J+X5#{DnsU*M<*fTy-iKM3+75?LASTuUu@29xgMDGIlvl_yP;#; zdeZ6S&2gwrxIM@EY+}p$W{b+ zmqmU>X2P0xcjwIJ1(U~ zq6cbYB^pO827r&*M6|=L^z>+hi}2VIT^b{l8r>pd5JfmS+7UioxOyO!#VV_HpQ zwk2gq*m8DpGpG!VAfTwtiM%X0q*^PdvZI2by@dBFv)$l;@+dh6HnM5pS|n;o=-SVj zEfogEayesT_f*O$5##mCt8+`63hx382eE}1K}kJz7HMK;R|zobg`Ho#P@39W8Z$Wm zK@f5-G5)0-_d`Y=TefIp4b{;~Ndt6X4UQ=IXM`qB0-RgA19csHTz6`S$?N9$W&|Ja z?7j(rCEW&IRM=|X?&~{G^VH!yUC<+mYYH(O7Ruif5sS8DynJGi^vXnh0eU3U!<~eg z(dFCY(W)Q80ZFRFo1L&5&;h28oRN@;Xq14sFz1^bOpzBtDokY|^$L3+=tI!}Ys>+s zQZqS^YWk*jW5*c#F{tMA*GwOj^cv&@oSt`i0(WJb#d&^gMC>n{1QcZ7^i5@ z&SG(fuAPpp9}vBpoH1G)6h0XVOk6HHy~~ox3I)Vlt%j{`27Zx+k+t*ONwiQ;VuLUk zi*ZYuLtOGhPOl?*bIB+I6vr4~qbmpDZ}w;RTc_FeOCIl$6SCfPy7TxSS%tYC!S2D6 zSI&dMt&*7o3d&LDpk_Im(a~{kno990%eFXX#Y?LunrKd!g*^mWPc^y2)tB03C3=r# z=2SAAykIBf5C)FRopmCHDp%ge5&xgRRl1))X8McORDu-J33`icr*5Y~sSNMUO#rTk zE*@n$+lLdPLRuAfL+_`Q9;OXsO=}rN(;6qRfv>8d_zPx$Gf9N3TkS=j5WX|6q1l_j zx$tPHkyK%UUiu7n=j(yWddMU$z z;$rC6T!(BHfzd9#QfC}nuSf+XCR(9QnHF1SWXUlv{*AtwT2eMq6WIass)0f937}^N zw`AMFc4(bKNtCZ7NJnRco1Vf>Lhib5UeomSElqa2UUs-6HRsgsrNG-!;(kcc=(0ME z$mS!3BBwsb7zmhWT$MMC$sWTkX=66&+ErChD)@@)XuU~!LqufQSgFqC%t*O~{PlQt zs*z`jCT&bX!o&$Z3~dX?*f^6!e@^I)G}LO&Y6CvxEat>%>Q^UH5=N@MS7{QWTY67B?)W!Z2J{r2mq6nlWaXhaKhU+}FDX7eugcBPIvWB9fIRcli zO?{by_krXqevKLh`+m%QV2(T=`ug)*>-KM^>EDp^Ro`s!V@#WJ>PS>;`r+g6L}YtT z#Ko~v@HJ^RGw%YnGSg#-)ON9SAwG^RLf*=m(8!~#3fbes*&NW@^qV_+4r5`5v!e1y zbr0P~TXi+UrcR@F-aL(h<3O(q!fGYO43qwI*WQUqioS9}pY*7g-QX;38B>i;W10!0 zSk$}+Z*+DRJQWgM6)RGr;p1wV!H!3I;~*xi{lK#b@6;J;m1t#mE@I^zH~L^*>LY>C zW=qkRkoeZb&G6X0%$Z;m%@K{mDv2WEX-?;JlB>UY68CX?b<(y$dG)%d1IF)Rt&Z83 z6M$#{ODZ3k{dZmP007xD@Y%@JiZi69>7~>sw#amvvhzbm(kWO`d~SJIo6ooA%)9>P z5b_Lv4|3hmq;E#^gvXFKWPHSRxe;P7)?&yICM*K zwyT44mk-gVwnHH>SO*hl;ZSqHqooIB&O{;wO$zyml{bKPdR+xEhdvG6_ipxbdy=0 zP1;{KF((Fga%CK1G_deJfl1TJ>uJ3p+acSf2nFVZrX)hbNnBI~bKDbJam%NMM~|A) z0Q!zA6`szR$Hk}uJhR9uev%arnT3@`R$((+uDQ&an&G{DRaaxxd z>Ui#=$S1`a5l+u#AS z$zoli&rp2ypri;UvLU5mb_(JTW`=J|;jUcQZX{{=(eVKuWulqRCWI0T#~+9cF9h@h zz$iH?6rr`opDX1qq!MgB_v+toU4JK((ntAXD6HzB8v zWE(X-xw>;*i;O~@yk#m}d3sLPA$~hv9n23-Bh5>i_X);a=ObyTb5$Ax`{SAAKw$!) z&6gA`aF7V3z}f(?)U+O3AgyP*2F%KynJNS4Py^j0FXAs|$hrcdn>e!c6*(awGynp@bi!XQkVypF-4cc46PX}6do z!(ov!_Rp3JL`15(BO~XM=xR<_br-8LRfp|;)#~$bt=bFE^w{fqw#Z_*KHTQ$Xh$hx_X_l#F2bO7{1rt4cL|3cZ4JPb>M%V) z27*}hAI{{c?vst!N#9fiwfMk6SQ8r&>R!&2$9@7befeg)*T*2@9-p6|AJOVkE5+F0Mqy~Dr8v83z79dTym6p>*KKygV-L`!?xX?`-3nqpnRDavak;h(fR zR&`AE(5BgeC!}9P<{l*wuywXCU>WeYK=$o8D*{IXvSKu37D@cb(jI{75gLFGR z*Zk?KhR#lUd&NK%`;uEN!cpwKbc{|2Z7g>RWCsfkq+$SV)mALUvv55AVtlH7B`{u{ z5V0;eVkzXC^jtiCXy3x?y+Zv}S8ITT=12&jUm5kf&@tEHXl#MI3A{L0l+MOFgrw(( z&mj9p_haj9xqnqm*LgouwsO8@4hT!cuNlRwj$hX^ZA%=$N1uKXEnP-i=_EulNgH{S z4fkZ!;~!C-X}=wVDE9+F0?yaNF6+|IfbE6Ms)?02wef6oE<&#pe{hthY{$7q{sx$F z{2y2+`-7{e1f&t_;B+W1>@?Fzrf`@Kb(5B~e$QRJ%`2MY5xd#)dh;+1a|~m6olH&i zu0t1e^F-f*{{>Gx@Z2#ghsM2j zwJ5zYzFWFVH2FVOXEAE;;}ihm&GUPP3vgUeSG{j`wbqNx%Q|)1GhsU4^kt`~p^~4g zfs>?EJ6$QKCRSwlRYz2SH`+%4yC8Z_iQ^as{pFI{NP{|>QA4K)2(D~- z-2Aqyh|Fj3>pYUUexM}^cOFODE-?Rueoeuy>u7vO9`yVRGys$Vf9uN(eiI|+lJjJo&n0 z(g7oegPwE*7STxks{_9fv>rgJB|-;>gciwCJ%tD!vhvFF7r31yt*T~Oc05&!FDr?* zzxf68WwRfe=G<|nU*I0eX^)wnp5{bE;K`nY&Qw7G=}4TFZ{9flbH14wK8_t-zf5x8 zf(=ModBw9ecZ9#fnSyx+24YBaY0W$NK_L0{6VK1_FPYr!heju}WWAY0$;)e+SU)#~ zB@PP1nkoU(Al7k!d=Qy91c{DoXdrIRzl=^-uU>a-MWxv(ONn4c11dL~WJQ(QQ}W@xv#xO{GZ_ubn91&Cfg zD|sY#p_VmB_mctLHHL)_!<-FT_`vRLFiKKjN(PIItpD|8{r-kT)ykXmfJ_cgM{J&( z!>XRizD*t~Bt{v50ZVgrMiJ*aY9y8z)f$AIzueEizg|6iecrJS546DPiI{=?F9S19 z^a3gf0d0%pV9b!TSR9Ux{NL;2Z*B?M!KWMbbd(x*7eagX3!$mrQYtQ6)KVUGg6LYFcO z5toB)b+=lwvY@DOHvmu&@g&Opk0%H-p7w-;P!MrYSaq9FBh3y<5~q+laJ0ZP!Ab{O z1RMl96gm{1@sIl;7bSg6mG=mIv1!1{GbdwNc_tx(7Ax~Hl+SJD!Z_zJ1Z=6+IFg5* z2k{>}`djy``es%FgNg~+EL_V;K+r2Fggl560-N5U8!vRmtK#rF0I&V?GfS`;S%c;i zIdBMIr#M}eLSSykLtc$+*<@1#CL+OdCZFU=W^qHt Date: Fri, 25 Oct 2019 00:34:06 +0530 Subject: [PATCH 3/9] Delete train-labels-idx1-ubyte.gz --- .../MNIST_data/train-labels-idx1-ubyte.gz | Bin 28881 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 neural_network/MNIST_data/train-labels-idx1-ubyte.gz diff --git a/neural_network/MNIST_data/train-labels-idx1-ubyte.gz b/neural_network/MNIST_data/train-labels-idx1-ubyte.gz deleted file mode 100644 index 707a576bb523304d5b674de436c0779d77b7d480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28881 zcmV(tKX>KiSVPa)$b1rFQcrh(?VtI6B0DYYaq9nO(M7b}w z)cJi1<2^7-t9>rv$2ukcxy~K`-XFNV)-rA@ zrIohpStI^>%v*aubL>6ZnP=@jR=szX9+%T@e9gGK+;(S;A8oz$_@)uEtiESlJwMu7@9`tEthTwe_>d9zGisYXZnn(3 z<{q_=GENzNoxRH4bH~rE6Zbz`yQSZ?{p=nee&Z%aeET>zo@zX@cx>?janr5exbOHq z?lS%`R$O+xs`!92%8E4+FDD*#ysEgI!)3+=pTYv!@%H2C9RBs> zPUb!Fk?}s`LB{>=+xqS;)muIXiC4DIxqHN?mb1?CUB#{O3ww4JFRLI{cD$X=FT~x6?d^~FI-DJka)$hYI{7XSllBdGK$}bC=k1Hu-z|fDHdJaQ#_>=3#7-6-FwIV z#|6X_DmR|mW#`?9Ii07ud3v#R;-hbg7dqls#^p;^tOo9Uv5(`=v18&#c}hFiLELvd z*874+d{lg7EU8;pI)s))cB&nBXYk*aZT5m)0)#s$QCxhq}-4=C2>obmYL z8OAlNRv%w?8jTLKi#)617V)UGZ67DLtbgbXn0z zcz(oL?uj#F69bJsv9Ww(Y{v*?vD8Yefadey42pY+zl&@2Y%LjyS+bQl8BdRp9_MIm z?Reldel4CvME3acI0GB8DHeCE(Lz9oQz@Prr%HTdtiKyaLL4jceqzhdh>qtFDLC-_ zloKBiZ!Mz5i5JJ-ibxu3r5Y;YMNEP%aiFaDth0GUBYuu!Hl9~JocO3XpNXCE7&tvj zVW|w>qxT%qFdiaj(2AdnJR-g}Hh)}xLwAHgXyh1hrzR8KtHxS-1Ka8I#D zVmTBpgsW?8*N9hfRPh7};c@Npl@V0p_d3x&j!uu|h?23o8|y2Mti?GuVmHOk*^`wK zf7#;UL*hKmEXRk%7sNe99B%QMoEDKijKlTC@5Y^wF-5E(#fVF;90Q~W z9EovFapJ~qCsxKKo71-Wo44 z;$XysNlfSXB-e|BE8fB4tq{;+pBD3_jI>-Fn>8Y;Ycwu{JB}AQV_$Oa#`#MCWMA>n z;<3z;aiSX8;`%H3!C^`9*Re0-Zcgl-lS?-l{Vokc9@nVKAaKkhYFY+P}iYOyim zJK{CQmMSemV#JHaAJ|N>W(kb($jDP7C=uM_e_|a)_#z#Nzc5#f2pp%1=~k?!IPhXa z#Eryuz47v5A;ojNZxtOfEHf73LPYhrL#}=rF^{Wl z@#PU*3;^+q5uW3M>Lw<3Bk`9HE|e5!NH| zj7#BaV*y0KjYZZ8v~e!2$Tur{EmFQnX-T}|Bh9=IA%U}#(;`j}66q)#;>I~`CRfR> zj6W4mxi}m*nN`VuAMxRM9>k+1Hj<>p<;6Y5R}v-SJKc=&z9X57uZ=ewuRp>gKXu{| zjrSG57}-`FRB>O2*dC``BwmqS$H7zBiSZSYl*D=<;fgpHZzGaJ4$eX45h;7bgoud| zei~&@th(hMigg?j^)M~t;6^}NSL;PQl z?acFyODw!WJ~I9s$Ne;&IbLwQeNNLj^gVq<+rp4=+IHJ%WsZ5-|7%&2V|g?Bu$*!pn?hto1%Lm}Uu)*SKlBAQT{ zSL#)t{-OMcGclG(TzxYcB!>93$$tjHD1J1aZzS5p#t7V%zgW~jh1+V^Jcz{*t18ZB zK0mG}R#4}t8U)Qk{EnMAoE@>YBbW^m2Qt>U-MH#P>Oige~u_#c|pD%ROR0 zP@GaVuh_DMtvHCDB%+j#v3%pMBVG2e+ngcc> z^8~JmRWLdK+0>D)QStGV`JDJqJO`E`+w8^%@Sw>uqS|8T#3hk65CVv0ykx-)YWqr{ zWfNH=EGl5DyUmL=au!nCQr(w@iYdN3elPwuUXu8Yi98kS0D9_VGjYx?u#gDbM9^W{ z88N96`{SS3m9ecjcDQJvjsh%DM5*G!H&LaNND0iu%IciYNJeB=g)|jRAoiO%3y(EI$n@-ugyN8l znWx6f0a}buHcdbxNEXr?iq}S>65A~jC{w_AuI$!$RIyvZg*G4%B^(PS-tcD|MUo z1b-J1m_wffFs_{)c3fdrLlHZHPN)DaE;XjS$Z} z1p|se20%7E?u{TsCQ98{f|tZTj($K_LE(t=lS+-T{Dg=@+Aro*HW#bc zGs4x&T^}>bB_`hz&5qCRvZA$Rg<$eIT&01}eW73y16_?h!~V zb{FU`>mlwh%HGaHiW>oB;IU6qtP8G7w%B-%@zBWUV;L^PYeY$Igk@U*-3btJw_;aQ zEF-b2YXwtYZAtuC)|*3oI#Qx(p-TlGVUfcoD#1v^8ns#Fz@5Y)U`(*&*iG>|jvqcv z2QN8qBm0O0pW}W|M8~CoZ*cllc7EKNXT~D_a5q4)Kr@syu^Hp*IB;l!@P;WG;)=*! z$<+X(DCln1?OP0EP?H+Nw{L8`{9~GvPf`oq2)AnhEIy>k*4AgdL> z+JFjT1B2a8woMlgsBHOIZt$fZ{_$XZCo`31LXv)15f=9e3V6xI2m zKG~d^KuuKIK&YNb5rN4f4k$>J>#X6(JIFT43@-;+{Gqbki-2!2JpuKl%$u7^sfYq3 z+FQ)uubB)%oTwd96`T@y9Xted3cw`ZC4dWPz!D+3X1=jbSxd1VY@WorBaw~A5>a|` zPSUSw)7BRGd;Dd*)<#xHjOH!y9$IxjU%O(QM&s2P%V;G&XFYL@y|nk{wc|voG`p|s zk|OqI>Olz={}n$d+DNe6ps?gxrkUFa0I&owK^kNRL&X3LLk#3hjs$&Bxow|8Jjqyd z6d8rgnpIw`lt6w+Qtgnm*X4_?BP!XfZkwpPS>vG^B#zSJjK{$t3xog;8Cy3#g1$!2 zw@LYTDb`}c$K7##LyUoDL!|JrelImjtb$EUr~4CoF7^s|SS4?1a83iqx58k2e^B2Slh%&KM2>Vk)hUShS7aC%{D9r1xeBGaen|QF#%XjLpqT z!D+?@1S1s3>B}=9yvDUgd?WK1JkLU^5f`>VV#v!r-GP&*F#7$6E z8YKugybFKvhf@s`2+2Sr^a z&H}||nKl4FuK8@ME_FLMcli-?^ z8(Eb;RK2@*x@#zY()|3`OJ^xk{YTs%WYDA-9A32kPWI?1p?%KO(@oX@=`$?_3KW4C z*&(cj4(ds_f(wWRAF+u#2+Rbq{Vv%MgY)WumNnn%5-26YvPt5zScY~Vi>FKSRhCe2 ztnr8vM?OHAC-WKBy3B&%&Y>Y3tF@Cs40h zVpGTBjJ34cF<^T($_V5>2$ECkuWr+~+ntJUSU?nFs0mSgt!AXHqIbjK zOL7iYR}(n`i;L~IpbiXz(t)x-{tLMOsFqbK|Bxu380>MVC={E{FPaML=iBdtsw`7*X*>PBo{U`v{^`m7Mlg7!iN0O*jGw4rF75-MD+sO3n>; zS*0Q`)Rl#NgBEcELIs@%7nzjdws5WM8k@U|=Ewb-L%KX^GrT?%TMx{(PoJi?Fn?bPRUy*(TX>u}txR^q*g}DC9Zu(MzNuzK&xp?hPmdsJKv~Ud}ZLGfh|=!iunH z0w`q9ge1lX_;t{*m}@z7#2MqR-B0~*>>HIaxsb#1+q_OCmyDSS!lAhhzxfny9MZL z(w+l$J+vE1vjOA?JhASh$XlE=ou(Mnu_%&3a3XJrlMCk0<$UQt#)BM!G!uqgquOiG zFDU#7e5?u@64T~37Xg?TYmvYmK{yuduq-V6>a^WoIA}LWId^ZXP8e2h9Mv{U&fP?W zyLOlCB)FU)9xC?E&G80A&Wvb;zzui$~p}oh(l;|9z__WPWEvF_4H=v3viKud+jEz9|n-s zUvqE36&oD50F)2IG^-KkVhdMtdWoQrjFN*#F-MU)*tQXl(7Oay4BepHsMLZmm9(qeBsg8|LBlR9aEhHTqc10{N zAU88>z@@myF0Fz36x^5!7_wrQvlwBP@DlHutZK;2G!Jg#U)P4o?TOQp_Z;i5ZWTk3 z%S$Fv5erQ8jYhmlU?o;nc1WaMG}Jns71nkoxWJQw@pK5Qpv`gM`8+PmjMXm*DRvUH z36~r!j5zMOh)cfOXu%CQl#@3z>CCZnXc4IrECHSrHBgmgOLq`lh0KQ6LBxs3POV;r|EF*Ha;k3mTycoU=aHq6 zFn4$coT!v}?4~9S`P#CIXh2lp#Dd8O-?Y0b$amS94d}80DRd~Wkcq2nG4@-8ym%S# z0;rFT{?`+qSk)dHcFbbpY08_v)?&QZC4n|pU(0?TJtWZa4PlxR(Ry)mzNUf73eIbbTODvG=iG5 zCs;1T>U>@>3<1UDcU|)yTER8Zv~$f()^NdbY{!;>RZyaIfV8| z$Ql9H%*guVTnzJO1JJt@P9fe!x7)K{#}ULk9vyKvjv12reoSy}{6bTY%ZbQHZ-oNN z_s5AtJB}Kr!8nWO!bNfMgcf?^^rj090DFRJExJ` zL^wURH^G8Nalgb{zvf@DnPY{J`gYNvVPrA26x~V=QBO+j>#dmlxKcBU7g{PJ7A-@H z488+cM~UB|@N`INaWi1tMc5mdvQtUH_%`X@ENz&!#r7`SDc*6DB{9iZrTpJqWIo(T>!jM>W~)zjlog{vIKVIV(FW7z``Q+IZ_9>_PjX?aNeA}%0>hwibWXY zG^YZkzXH+`k(&FUNN)gW@t+A*IzkM94H#S{E3Xn2VA)S&DP6Qnc-DTZy(Jr+qF-eXvWM#0VPN4>Jwdw1SBoMn`k!1xh z3sz3s@$A@k1`G5IyMVS>RA&^tHmxV=pG+&s?_P)9Y*=k7Cji2!J!J>MGywrFFwG0!QIsMe zS+Lt4-@Vy_uw0vVlilvqoO*YI3qDKWK!|Ipe+{OYlNL)gXUc)W8ZXyo^SZl)i(p>+NdF_2S6a#tS1w+@Zk5p$wVM61Om*IqN>oNThJ-9sb_#f z;2Mzl#zI^a9r17g8SHdHl)Su|$u^I(GI9k_uVcU^h@y|8PYDr%HNcCY*rOZIWm*vo zc{InI4E*-vNAU66(=8)?5fwdB7~~sC%0J0^GZh0~0A__P<*^*gbdkq%K@_%}5OZeo z34cPN5*1cNQ}iscv>~cP3S%8@e$@jS`2*TRB}=e6e6CC3j%iHs8T2j|#tMT|I8v4K zYU5)Qn?^9@+Od+`fTCg*w6Xzzd0G6Bjca;S6oD{LX$?`Q(xy-^M4L6T52%N#CspLA z*CD75RtS_)4*Cn*4Ymht+eXC$Zx#N>vZ)n;`gSR3RG%DB>Lz11K@O+eP*oDyZR_s; z3Ts1Ju#F4fVfps^PycY9wk1NOH7Q3RBnw*1^lrrgsyRCCt4S|m5t557C{kYP37GZ7 z;3F%ChIJCWy3ah4LGo-w4k!}igMc{tVy$hu2c*F)d5%f$djJ~{@hB|GG);#)NIp;+ z$rKVeiE+o+QB+ATyJ@Lp$}VEFs?Gk^6!Rl2NAX5}Ln;TzU2cMU3z0{>9&%PFY?C5} zwMp4J?RcjsktlGvfFA3Pc0ha;93V?Ys+11J48kXUAnKCKX*shyu&ldb3slSP_LL~| z)#g_nM^UFE*#g;zkA{+mV=N*OVTA5p9JW_>v2X$;P=jc-YsZlVdL2Ikr&Kwu4NP$A zscdW}TU+Xew?xSoPZOdasZLKW2mEt#0jRd!?)$O^OFqH&nRf*wsd!KoAZ}={6M{HQ z{SE0Z9t6~6b6FZK6R^M10V41rxdX=H6t3vHic%U5T=gY%NFp{ug&>}eW}n-B9XKZ` z0N4wPCD{*KeH*{rZ?+Ji19HltMw7JoSTs#EygcNXmt{{=jyAbp>N%zdKj%wJx&xn$ zsuqDB+CF+rWPpSbB1ilU@B&pTu@$&zQml}9LkT+Ik&u@V)}Y$rR*NL@DNs8+Pyy#y zEd{tqZVV@T5zU?=+~x?bcKhgJc>AFqbaGkE$ot@fi|kXRMW7U4LRLm|+~^3eu&MX5 zRX>q*!V!_ugNiQxsE&&5Xg5-g>yQjYlqbg`FMvOS!jI0Qdc6zJ_j`}&?7NQjtM*`$0i6fu}&~fNe(J_v!no4j|X%lVXV0Xwo zkY)(A@UhGNG|=6CC;_mj)Tnmf&QtEMn{f|yCA8g^nFY*Ko&ec?$y)Bx9U<>QdoI-- z{PRGgL@GUnO3)!tI84$hx&^!|!VxQ&JbI_$K+-kA0z54%G0m^J&i?*kkt;!Qb*eDx6z@V^43i6jEAYvyEH0LlT#gg?nWqw;@)LX z;v;~T0LC@x1Ib5Idl0XKBAVKSr%sIMC$g7*wi4O2c&5C#@zGSK=qopXJ1%4eI+o{%+B*q(yoeTK` z{{j#4T~oHq1)k#?Mh8!mSyeoLD$sDruA##h;yBE5d2pVY*PN`|e>v7f$69pTkB~8x zl(-3`NDB~YA7KaBW%YDMU~O+NQM3~_5X-lbz>xkm)OHBS(PU2VT|f}pbz$o(n&0kL zHfDba-ISdq$4@pIF?Z=J5-Sm6sAxIN&d4MIF6i)WBjAFpID8@!zG(SQVmovb7A-X- z&we1UJTsdGV5yzpwZx~v$d?qg*%j!cNo@-01>K;Z60nd12x7!#HF5yx;E+v4->%%j z>z=Uuv6Ycb0AMnnQ4!2Ru?&BYg~(9?F@Tc~rr+ivciu8Als{Mt03h1n zjxRu<`T0Aq;5{6+C7+$SKC3QOyPkW!y4K*r`u(?HpFbSq1~%>dwu zuv$uRG=k)Ad(beE<23)yh!^x(61UtpgR|N8aYvw1>e2fMM6LY-L#c}s(^t9o#Jlu4|NIMmnIL9 zcA>D=fFjyB%Rp?eoMqN8OOCenkxOGeAmW1CLbVp?k7av|eJ@6vm(`~@O7<}XW>27& z;0{|6ok=B1(6uH3LsQhXcc?@D3~Z2QeM_7GFd~he_n}~xdhwu8s6v=q`Zf*H9yTt^ zs>oheE=-?!0K!7C-LffTHDaRyiRgfuu&u6H4R%cS!R?`41LcyBTAp^X+k=@ZGC<3G z7uKOV^dx-Htb>5bAX1*nYr!Ce{`6$0(b3oOLzYS92o=-rGT34GP>haXP@!1^v_RR4v7N94$UfY)^L!+T z;W9;eN7{Ng8ECalhzZDD=<-mN!}cT1L8l3!cMDT&<9{zf#8Zx)m0G$9qgvq#Q3verp9s*^r@NA1Maa>AZSDMR{MJGGEyA@TFau-@F?PO zL+rpZK>UuPl;%sPzUz>5jh|r8(gW!;5=G-QIY`y$$Md(wQoLc>wTET4Tqoj}Iz(6Sy8lewLrw;EE*--$YQZU#I(2Vs1Ks__H#gLDP ze+KPvRH*bbXrgZ&8{iyOG^!X!_}_qt)a)Wpm|YxqrAcI)x@}of0$Lq5VWestT!?=t zU+C%q^`!$8B)BC%GrNjC4Wz(-JQ;^;{x4~wy`yvKo3f{heaEgPPt8VzdVstzk%9T z^frPB5Wy%sK|r@B{{11b;7T)9l%VIbFtVpc!kSTdO9a4!4y92OWW!UXjnwcpTMImS zDzkLE5E-8`WZBlJy*V+dz&WnLV}WwmvGTqf`S`#HYTF-$j6oGc8@?$bz0x+#KnH1sQ~HH88ZAfYned(@mi@b(2`hVGPA4 z9`_{EK`EY*ohn*-3uFhB=da&%RS|d+nt5x3;z_y1p_|~j04bD^cF#Tq~!ueC)@~J9QfZmRk>2|;ipa) z#Zu32bGh5uqc2*N6~%k!}Y%JpGVvXbntb@Nqa)bixi=psZ@9CWiE0P6s# zm|9WON&52~%geKbSis-L6P-Bx&_EEBgX&yVdWt|=H7`ZKO|>!kS8G}tfb^&~Z8AV@ zY*4IqOnA(Isp?5MDOb2G4rJJm0~beID+Il*CJwlSWU-}%K=!;nQRt#tCCH*G>gbz1 z==3){uLE0>;mUiDUxxFfJaf_?#MXdwp+D{e=_u(VD&#>i+;Q9>fFGm{6Y9;TKgm{K z81Mm|*@f>ZextUJn1@pUrbOR|@ z`$yXfg`L>lv8fN*>VfP``FH1aYR>a2!1>LrX{pXHGCJv0#MRLsgfUHaOY4!h4?;zc z+5fh|j+14C4X%!ZJiemSY9#i-h9I9?$tjB8pqrvHRK7ReT2>mblvRAui%9o@Bx^=Z zZiw*rCb``Vp;}p*u`4!{^X_}8u1p0%!MzBKHl zCz-AGm%W|m$%|tsP65l)UxB&24AM6^OGA+y@;)TP@&M`hp?ZUtrq|4R25Y11|AlC` zklG^K!1hRF8At$`2|XFzYpPZPgQmV&6eYvRISN0Cc5hmIW^x!!&;M|bD!7NRXilj0`ADegvcpveBS6hLuC2dPeyM>c(HbBAl6t?Ap9zOK+Bfzp_K^VcPs$MBWKG&8F(}$=`vGajT2$VV9t?=PG zIu37{5!L=0xm}_I>vN9=@_+6|EH;@EoaaS)7zMC6FxkhEG?GvO+YC$%fD6f4JajZs zR@dMwuuu`ik%6KoY|=alg0RXs2Zo|yC_B^D3D+Y9&se~aI5jhv^m?d|9i@RF2|8@~ z)C}-C^eLIC3)QX%ktM5ci1|x(HEJF>^D@QM|KaieA-zQARZGlJI+K04@`1?TJ!bP?2E4FDw7D@ zRnOO-UuzLNovu^%X=dP)jtVUnRO|YFB5RhWM>&eSfHsAP;|*?Qe0)cMCkujFa(b+y zt>X$R^Um#j%dae^=?@+ zJ&vr8I#s$>!p&@L=*?6t#dv8b{m2MA6cIH_LYKg(xQdhiQer`Ap(ZQdBsQhX0*l*~ zmjG~#Ck6xtat9LQy5zqSN{&9xf>(5S0fjkh(;s1;OhAWxyXl96bHW>ezSQ*EqZFgs zfesEO2{rb5=Q-yzejVre(;My0OvdvpU1n0l_(P0fjj4gr!d;(Bt#PGS&{>5Xh}3)1 zu6W%~U;^CTv+16gE{_c(Gl&>k6WPD2(U&T{Vz*)0QU?P^xV~{gL&IJT`3RdZDx~KHOvCvA-p%<; z&lJnZWSWxHgYR$c^X1K<(9npD#);48l8?!GA9e)v)?p`Aja1eprvF&> zU(%BxiELfRL8RI?V|htRK}%2Ih2jBG72wO%pn$4o!EOQT1HM_lE3L?Iwj~|n2v5q@ zk!eeJA45r&pMLVN$WV9DOKVH#y5SrG5i2^+B%fIF5Doy-n%3R=q5wY@ig6KqD!s0* zygh~j9L8kKWcH3TY?Ylbvs;gdMA5xs8}(clXJra@Fb5n8B?(koQj0{lk$}_s9@zH4 zjno#DumX|ib{W@nhQa4(en@bfG_iWxCu#*ujVtouF0RH~4wPE*H6Qg8{bl&Wh*3r| zE3&_q@v@4IlZiJsHhvc{z%e7(%6iNRs-XfpAve0PRiqbb3mAvO1$aqQWU681=TaF( z^>UMqr9AxVzXF&}3mp^Vt$Yi6b4kEFu+xXytNUK4-rMY-SCO7mtkG?y za?G7MT~@1uQM3@YN>TYSez9!rt5JO0PMlasSOH);S*N5oKz5lXq9T=$lS3T&d@=Q<|)Y`#cxR%eM3EoUaQoWvkId+Abz`sV;ig#ts!N zB@n&2?!*Z0Yv}@sUV=l-GlVzaQV2;)0t0@w{>;t;jVUtb!9)`{zltytUqEXdJ6+IH22Cd*wF=MEf z+CoYgW*|V`He&%xA^%DlUF-wNY9@xflI&J4B_mRcDWsITUjP&=LutC|UJZ`fVDjo! zL9YwyD^)H``nV-EqeF(S`seWww)=XD<(-o~t)7bmAyg8=k!+9}(-|dC@_dywJuk$*}IMM=KuGvc+a%DRFNfGA%twdO*@_*PpIGL`yRZ!jaMdzI~V4G2)c111)?y5}C5 zfp51I&`Ytk7Y}0I|u<5sY)GILt+a*#xWKqa}=#y}q zT2Y_@a9jc?Iy(;ngM@%OcQaPB8|Zm3Dg{^mQgR<2rz#ikGqmiP!q~pzffu;vj{kbu{hWT8#olR5|t~iFl%R^1T00XYa@C zJ$4xgDdeHDW+l_uHA}Ii9WibRgn&eML%zR;7nX<;b$~5AmaH;BWMPl#gDol|pV1LG zCTr8y39g{(dz(PE*o{u@Q~X66Sj+)@C4|flG9bGf6^EwAWgc8nD1 zzg|^^UgQ1I=clgZSa-e4X+|j=qOY7?E!+zw$k!~}r(EAON?+! z%9tb{fuvp}ei)f(UfE#Al%B?!SQK-TrQOn;mZt1avWao`t9;0nlk)|!aj=$Yn~A^b zq-Az_>?s{lmgJL{=QBK>>*29dwgm;pmNWWV!IR+hv~=7_Fd=7blE%AjKZ2efGJH6A zy_F=P!p=MR5u!(oj*uKKEl`xy<57UoQBzwnv2(j-C*vt|rGFVG$VBIl(*-K2qt$(R z3>yqnjZWwrPPf4T!j}QSOcF1GWWEQ`zvXbMH@N_ITXh2}BbGZoSkx#GB&Y&g=2}a7 zg0_3fLV65J+w4*ytGQFx*SS0`e9jr##l|1Rp{gwt4JmT3v;kg^6Vh~SI5XwoPELct zv&D&>Awv;K-_F*OAEuijpN`1y=nMn-m?x8&-X@1O~|WWSGJ4Gs#M&_-R@#<_ss}U?b3`VIvB$NMB0U-A ze4DOGv#UJpB~2|;mc2{jpx*SL5AdGnt9+JxI;ZsgCZpUpS0kN&k}+6)GL8s=O2iY} zEEztYu1b9QC|cUn?&mxPXGgN6U&~PkrB;nHJ;mM#PBE35M^p@>IY2t?d6#bocn4^c z{tA&7E-97v(uV|pm}{b;r*!3*Lo_&J@pp*@%#EO7)4(Aqi1Y@3{6~Jgj1#t-R;&y-v!=rs$ zX~O}a-5U3u;NEGWSM z3oo|IDDT07kqyLOk?hO8Ha8&B(Ye7fV(df;QBxlEg|t3uf}fHDnpS*`x}X|dV76~f zjayoP(myqVTMY6xA-aOjrQ?|=nOlt_US4U}2QqJzJ+HG4u#%P1Ob zy=_$vd2T2Q<&qxP+%l>_qJuICwPbEEX{wY>Z5(hm#+Znc1UzfxJOXZxhj`Uu z0M#~a5LByx48l-uQRqxL9Orus@ zuRAnc`#>HBP~k|Q8A~_}Tr#wsC1-{V3ur;5!{Tqq%_6dB0iv{!HdjdrZy#CPKESCT~u@VZ__@ZkI#UKvg+#nNGkWiN)mfLM4jq z4AsdfEOn$NJ>;NEy%23Q(z7NjU(e@T(zsQNLbf9MRLo4Z zi{s5sx_LypMvuAGu@ihXOK~;>1s2)Y)MgN|&zo<#jLvGJFg(UikrE3=8SEncy#R9= zB(ZG*lbr)rApfq~0qseW76?U;!QmtSIS*I$*6)G=49xUFg-QN$9hpgj%1#j;#IY0U7{efV3sn?H=8CDfwXn!7 zNDcHN{KHU{DGsA~=YFJv#l`bsJPi$j)rh1I&~QqPXPZk}MQwNbL;BB9+cw>@AmM5H z0}T;4QZIKcfSt00x(82+mU|408`2x%bUDXVYOFj{Emkm^+yQwT?+Fol#A{vA2g+j@ zsE`_uDe7}lq8EUfIuh{I-A}Pb9cS$RB|{@U-&I>Tl@B1SjNzjnSv`KKDD6DbjX#cC zZH`AEgI|QHu7f>97*uwP*&L2lxtcz~q@eU$Ntui^6hSP!E!beRmUnxZ?Rj%-#&eZ( zYQmUSl;h{$j^;`bW%<#js8`5@ zVccS5c1o#7Q^G_Angk5WyaVI2YFmw?uCru;TFA1x^!MqAvZrvO~$oWx}?k5}4OmuDzTS zeyh{oIrGVHHP(U;1p&sAN!14vVvg^Bi=iM`n@`0z#GPfaK$QcDmB82>C#I+sMF~{~ zi;W@)9Yz;4hJ-L=RkG!a$l-80D$}W|Hk<pg0``Fx;><4SKXeawZAaeM&(wYi~JH zig!MdnL?49wAbdBbFX(7_Cl?>yZP>tZ?k?r7~289FhS3HH63oSLU%%?GG!rsB`;M^ zGVfPnsJ#r=kVqsW`kd)Y@d9&l<8Mp}4Mspc%E_FB7qVAKb!4n$aJWMu&CPYH@({wFF$qmH+QVqiFD0t zDM0)nx2?8u6#gLBNz*|%aPr#ey+TT8PV22U`=-Bw{*(@Nu?WFx5I+EMN|i`W%#aZE z@iDuzC|eR)m_5~y4$!o=o!+uiNpzZqkBjuQll8o4{tZa9sLeVW5eIEaurlVHQ_1$<)b7N_ATBm!1IAIGg} zjqH@JnF4d|!E|~%%KEhDC^kj;&ca^jb=Zpr5B4s>DH=Y+M{SHIl(M6GW;<_I#nUF~ zfto%4@g^is4G*iixjr=%(adpkrmq8kvO&+3RfD_>8)r2S!H3B>Hv3iFSIOIRZ?RfcO>Ea8 znH@$hI}^yu7*>gRie7#3MZ(NU+Y$P4Rm_N}j6?cX+$Kgr)zqbNg0g6`gq${iYsu6{v}PVt`x4_=QuE&xw6$& z6sX8skf6Z+MKS|5aS(ej2gD<9>pbqX9t?4p2;6uB;B^HkTHWDC-Rwvb$J9eJm>XuM z;}mFS#e3}8kvN&$Ga8&cZE5@JKsat5z{@r~85Y%(bq85(X}U~Io#ARa2fgVR-T3OD z*5-_c~H&jJMv|G2CFMN3*I23}CP zr~YZQ{g1*_zzPt6;jE?WczDWAHBB5|RIB8M)94@=Mv{!~eA+&TCL|PMkZ)E(YB)-g zLh94-r?+CdUut$@ZJ>qxPQe+H?0eMkdQMUka%!p%Nji>GT}c9#tSV3(7BFx{IuI$Q zC=TB3S}AihIb@D{%65!`$a} z$1WtY30+JXIfHj;Q>CKj`KZjn3q|hs(8nQ>rS|xh8*P(nh1#@=rXJotcbk29EIPc2QxAx}=ipH* zTo^*+WS!gKTm>XS#P{ZBb$zMQ+**gD$5TrW!S>ad0}J3^uP? zT5xos|19}{TP3&YpxG6*0o+riT#2tvX{I;=Z9dueTE-H+2x(n(to$7HWa}xVC63>K zXjPeIp2~T+$5J1rBMSBR;*x87Ec0!>x9u+2JbDv@M?L{(BtFmMm#gTyO`&LwJ(1)e zhi0ltva88C&5#L@EKE)8c8GzH0b^gu+XnJe2WYpbvkOQRg&?}4TyS+shgAIaVLVYy zGwRfQs@pSNA|W~4TSFyTjv33>)3Q`qz{=9`j77J{EG%S6di9YCAyUScL?)?(Pwh(~ z>)?RsC|LdHEC*l%nSK#Lz>fCR;p#eA7}JiW6)5~Q%5a&RU0W1ZgghkGo|Faba8L!& z(yYR(D=x9hRnuf^y(ug@O3R)ZKeuu4q=Q(0V*^y^Gcd(SLp-oq=9^h_+fjwl4^?SR zPmH3ndWqs#^mi>a(m_vLiLI~XQg`F*Y|vR?_KV(MToc)Lofe`# zAGnx1aFn8)dJECA{hW6?j6HA=?kh;oGZc5asr zJ5azJ{sSQj8z^8w2QRbC03fej{U)Q!VeB<~%k*^G^X%$Rk$-eOu_{RB4BbUDPb*{i zrH%F^!`c%6@47as4ayPAP$PBq8&0|a%7f`oN8u19iO%30!)bnKp?JXLfUsYyHi9t1 z$*nk@7$2eUClm_MKa}gJ-x_C@|3%g342Z`}FXmmtobrgH4ZEi45#{}0o4`mUhrTe? z8pr@rfJQA0#MR(;+H~M-aH0^TFeGRI2#Bki(*aW{(#`cqR_IMYPj`#~-C3H*U8`Sg zJ~6}7f<&z~R4(zGuC`Vwm);^wLr;gj><+yhDIO;KV7?hp_aTb?54G~KAB$H8OdDGM zO!9Fdz@4HD!k^*ISX9hHr_$-z$PPH7rem?EB5GK!^Z^;JiDNch1b!D$7l}94OJKGR z%CVacL)#hD4gl$raGWX!waZHCI&?f8gqXZQq&&|LemiU0A!|C--L&bzi9~^(qJif$d{o3V@dbJlGKQdDb>ozxd77!0-k{`ilK$#66;_%P+29x<(pWh< zK{LC;=`|=oyM*2AIE>-E4Dvx>+mntQvTAWdBH>pKbg$+Ju)?Z3Q&NAK^6Pn_HbKsk z$Lu@5ySc|w2Y6M-p%)~nbPyU6ZL2owDe_4vGF{aBG~E@;b2-9$->SVJzK7U{+JTl? zNuoUemIg8?A2e=c>6ls2uu`Dy@Ls}n7c)mLirS2sGe{w}e%xv}(hp@j&9LmD-m$AOuXP9h^+1I-_3d`NG@r3;rQ&P>L|Pe=Sh z=5e?lI4uAY6*HLtp#}aSoC7_^)vMLy;|#T?>5DG;;?0hKE&#skq1XMK@PH<&X;lq+fmF5$0I)I0$YfkE3c64Db(|1O5j*I(dhQI2Jjm*!aq| z#lf#TmjEZVnF%!ci{_x{L+MITaH+1P8&dQKc8}}zWG*#KwLN9D{zjU$405`RQ|Tht z$BYdOqzZPh$DuY%ByiB!ZjMqY`c5ndT!~qgpTh3W)W7D$?(rNqmT1v=E<%D|D>1UP zZ)vIk1*V3kk2Jrqmq4w?Q0}3yh;dI%F9q4`Bq7LP!uHJK54(36p?tRRIA5nG~f?&++as zwvdK7vQ=PBd~?hB#qXmIJ$ey29L-d_TvtFxJUN?#>joiU&a|na(@KsF{|SML-B)-5 z$5Ejm@f_|)+ESGZJ*{z45t7K>c>YGo$KT1c@>I_|cE~tFTcQIe{?_o+VQ#dHtLjK{ zHVy76(HU5PObSJfu2e)92>fJMSFWkk8v%Fdfa~}x1Dcl)5Q@61P|-Oe$o5y(8rMlA zG&hG1QQOfbpz^RP@gtFT#i>8EnqN{#9)g*y^>qJhV(;mnUY`E?T8r!J?U~cH$pIRj zZx)0CpQEuMY(KJunL2s8A=ODmNT#Vnp#ybE3;r^#O?c(xk@Tv!c2Ibl#tS+onmHzj z-cqu@IYwze2nbwr5Gpt)ryS>fOQlk&d+i~dRMf~xEh?9C=9ED3m&@2&;AKkc0Ez=1 z(TM%Hg0?2sG|>riQ$9cKI&&R~^~SR4AWdjQx~b83(|QaeOGX{Vt0E~TPoZH3qZR=n zEGdnk)2Y$GO0Ij^(7`B~Zq$xYG~Kuo4N(eUGZ z=IY+g!||}E@5_#0fD0fv#`IO*%N1>-K4)#%eHdn5@L=u4;JZa6^W!yTZ!|uqo_NGu zNcND?v34S_tBEO83sPtHZtXl}+iUp+8U@@$YgrubXdT{=&hz%@ZOoh#^C~2n6wgD( zIdwAsuAG^i`~jdem3JV_K|1mxzaUioQ$ea zrdL(4h0gjAsYgz9!q{BYmY0T6g5zA$%+}es>+yNg7bg82b&+yr`;p<7E7?NJ!KY-F z+6Le@?jW$HFoy1ab$T}q2w<4)OlHl6hmz1^IxY2!vYz9<&?aFn7o{f>NBoNM=^~|G zs#C5a8030*dPw=kr2;dRPnqLK5Ews!O@^6n@Y(kk zk?-j(C6ZXu;qrCo%~D&6RK+QI_vs6qV_(yig?OxMGqobi5H##!y#$I|MXig2a=_!Z zY>x}S56-`nPuVE;BbiTtxf&FaB{R!J5QZFsGNhjomm3^ieatkGrmE3x7ORKy^=Q@h z)^CV9H)^NI$t6RAHe5F>K{;PBtbzW9%NI!Bw6NshZVV`7j-;YNKO|F{>O2C^Mg~PP zuz@i2JiO8pnB(4h=RAjk zZ@wi%0f-DawJdpz>-|hoL99>Z*29u|Wb3V7b#NpWRqVDjmo159S3`R-g_K?kdFon> zJ0@#sT;y08I82Hi7!Rq#mFB4+OdT&eNUG{XRCK!cJdG0FL6l`JdOYuy9_$s<52ZFRhJEWBb z4TpqoOqfn73n~r{cRWpXzR*nC{lp8BI%eynhnqRX0am1*kkZM^pwE=?$pQM;U7`q(F7h2p;0_WS;QEcB|*)3c-4XNDwwfeh5-wQYC8!4v;xEl zBpuGnLTakNu7gFkc?Z3^gv-Yd|GLkgs5&(ruk4+(o^{_-`$IL)jlzh6gjpQf4HDdZ z@_sqZO3+yV#_612c;qPVatHON8A#8TV`Y{lT`J+aQ~nY`e@=8FxQO6vbqz}l)eVva zm^nqxL#Gk`8r0d2PXkA)g872hugJ-x767ic#zvc@o9D1G%I3)7$JZ%FLbKri7$^b` zB9q`4phNjQ{->3GGUMl)@jHB$-=x&+tTlVRIK%KoI5;?=M*Fl3FzT^opCg=gKwQ#e z7?w%ptVi0_oTWMQovDV-v8F~Y`p;U4A%7=rZSvAEGa0f2r?xO0w4|XvL*`DA^1I}> z%(VS_vv$yGO@JY1NCazwsPN8yH3j3dGI2B)xAW(7Bi92o@+ zY$u>+pf>y`g^O=f4kZ&9>U54kt5C5Zvq72!#?RJ)Db$|MXN9>opRty3LM0{u*m3WA~|ms zu=_)Y6E8W3LC7G%GiL z0p|9|7CLsjw7jlWFY%;`-rSd&4krP0JqXaq=}jg;4w0LzaA4`~yxyWP18YqnK5e8K zF}13sZpok2Zj<%cOs>K4v97f6cX7>}sl)^%lHZj>uG1v5ORA=4le19DAI#ipAFs(^ z$vLA;7OHlZ+H?IIsSTD^#vH787o@{5I~lHnldk=CUWdaQDf#HT;Y?eV&dBDy>f}L! zfnl)|&+!;?95M0&ch~8U?ttDIXc4d&frp2-O%gYH0Vi5)H3!(El0sbdAnA4ARp!Fc zwI`jv{>~kD49w7RdCMY6_wDDD`rOKvzVYcsW|JPBE9Vy;o45_wSAjlYFI;ITsBP+U z;4n??wXp%A5P*0^RY8HqtQYYDj#aY6D@A1JW(HRjF(j3hc0LTdpaP=HX$~VMfy#Rd zL@-m2lL^3!Ss$dZdd`Tke%+<%={sIuke-M6ulUb)?j<=)d0ejfFyg71H0x6v9Odh@ z0-DF}(u2EX?@Y&>N}$dz6nZWO!s(cE*&XMQ};Q z!cE60GQNp}h%!!kv{j9a9fh7AzQ%dS2a?lBhQ%`!c#@71FchFGEdvhtEo#b!Ssn)( zu+R;S&;q36>YJWs>HJ*8lZyQ8_PAT~d68aOG5bwegU7})10@0>tTHy)mIK!dB0O9S zi`%Qs7Vi!IAN(fN(>RH3&32${F>s4a9^0dIStd8uuw$x7AT3O_Cm{TD*c+{6PB=b6OrA0! zqand{r9#?Mcso~;x_CXr9`kZEcO+PjI=F%u6g!msbR?*L@m*|o_YKiKnV(03zVk*; z(-8@8=u5Ui3cArH$0JHv7|nbLS=l?gvKgBG@!RK@^9TD%X7WgImNjK%O!Ha?Kxw~| z9D;(lsfM^t*q{~=OujFoDT6Yik&SkbFaAapzDQJM{Kas(1&gNG5P`GCY5-q`cLX1E zDpkYIsluAq>H{MjJ)#+uIdpd-HakxZk`y~!R^#nGXRE2nmqKDM5h?*i&aqC^MHcrv zmtA&4xwx9<()1^an~mc#oWrRIMN@U3Od&K5NEs}* zZByCHt~t)FJinL6v%62Zx}IZr@&>EU1V;|8egCOx43jfurxGV&;md|+iTTKqi)-K7 ziBVmZv8E6N`sGYCVw^<lBk z6a!0Nl$JQ?lOxn*x2BSDKfk4Ftw)PHx$()cy4Hf^XUNuM&9V1U#9bXxl*dAkqia=* zBR?RrQ%b@&YjQnFk8~FE@j0~VDzIWb%Z~>v+}W>!gYN`hB=tqG%~%ZAkn>Hwp*Tx^NjoC;NQU_&fub0(df_0oYrF?3<2A4kiCqIP6b2_*;3 zF&t7B%+|n%7|0X@d<`6=Q63|$zj=M41*#veSgkaXoVy6FV#B^W*a`h0UO70}vIp7d zj^87z;D9=ruiK$EarBu1b6x9c80i^2WMb0?+`r;;AEmd>l^#RfmrJK1G@w)R={SZr4fwNoT%Ej}N%RD@Zu-b%Q z50PPF7H6KOkAXhf;%mv^EA{}p&h1KVnBiTWJSCw3b_OjWvLlDB12F@e4u#*`BbZ| zE?CM-^G5{LJ533A=SMq2vP#bXqCPK;Uo#s^#HWb;ezVA@h)>RksgGlRWlJSGg2%T) zD*BwZX}TParj3$jFly8WlPi!;;4%S`0htE4EsQYeal9813>{fCd|nP1q|*;yP==`d zMydz2&wMO1MKN3D3A5rb0w&8M_6~0anp!bE9VMeummx$g&78UUCIgMM%RD)`BRe_H z?wqL!_9Z{}ijuBfzav{)BHr~?o=;%g6aC}P_7tNn8`XI*+z zHaqf;DbqOFSzE=FG7@rqYA1lsgKzM<1cG!@DHMV&BEB7i!6~%JRGeVa?dJC!o4&8Q z4iYZrx7ll;!ZTLk5=I3;%7MxD;DD$(9xMHbmw4+?aWIryVYIj8rr;=*-M3V)IKJoc zDQQ05-y3YNW<}J{d>SykstRwr+Ay!xBS*M%t%2rhJ22#BRJFH=U|i~ zj|7WuQ`b@jbGhqw*2&p~2MvAqcq46TIj4nCI(5xMEL6>?iffdv0Rr?|^IHu>~T7bLdxxyhFz0k@<<0Fn)*f!8w*78ZhZ;n4WCK z3Wx?Arwu5lA1S$&Q)C=IhvxKZtA^}MQx;AfoPnHkLx^{HfWs!6ECM_hod7pY^kC#v z$CIg`+)pK!^OUQMpXC{tnvQXDmQuPv_BXeNj%^OHH|ZzB%9E#)%Ac;|$DHh|-w>5+ zv0YB=969Kerad+lI03T+iQO17f`L#xA%bv5Kz4b~(%e$d>?C8E1R7o1v7kUNej@!0)&hSC z{fX=!>gLiUjU~8J+X5#{DnsU*M<*fTy-iKM3+75?LASTuUu@29xgMDGIlvl_yP;#; zdeZ6S&2gwrxIM@EY+}p$W{b+ zmqmU>X2P0xcjwIJ1(U~ zq6cbYB^pO827r&*M6|=L^z>+hi}2VIT^b{l8r>pd5JfmS+7UioxOyO!#VV_HpQ zwk2gq*m8DpGpG!VAfTwtiM%X0q*^PdvZI2by@dBFv)$l;@+dh6HnM5pS|n;o=-SVj zEfogEayesT_f*O$5##mCt8+`63hx382eE}1K}kJz7HMK;R|zobg`Ho#P@39W8Z$Wm zK@f5-G5)0-_d`Y=TefIp4b{;~Ndt6X4UQ=IXM`qB0-RgA19csHTz6`S$?N9$W&|Ja z?7j(rCEW&IRM=|X?&~{G^VH!yUC<+mYYH(O7Ruif5sS8DynJGi^vXnh0eU3U!<~eg z(dFCY(W)Q80ZFRFo1L&5&;h28oRN@;Xq14sFz1^bOpzBtDokY|^$L3+=tI!}Ys>+s zQZqS^YWk*jW5*c#F{tMA*GwOj^cv&@oSt`i0(WJb#d&^gMC>n{1QcZ7^i5@ z&SG(fuAPpp9}vBpoH1G)6h0XVOk6HHy~~ox3I)Vlt%j{`27Zx+k+t*ONwiQ;VuLUk zi*ZYuLtOGhPOl?*bIB+I6vr4~qbmpDZ}w;RTc_FeOCIl$6SCfPy7TxSS%tYC!S2D6 zSI&dMt&*7o3d&LDpk_Im(a~{kno990%eFXX#Y?LunrKd!g*^mWPc^y2)tB03C3=r# z=2SAAykIBf5C)FRopmCHDp%ge5&xgRRl1))X8McORDu-J33`icr*5Y~sSNMUO#rTk zE*@n$+lLdPLRuAfL+_`Q9;OXsO=}rN(;6qRfv>8d_zPx$Gf9N3TkS=j5WX|6q1l_j zx$tPHkyK%UUiu7n=j(yWddMU$z z;$rC6T!(BHfzd9#QfC}nuSf+XCR(9QnHF1SWXUlv{*AtwT2eMq6WIass)0f937}^N zw`AMFc4(bKNtCZ7NJnRco1Vf>Lhib5UeomSElqa2UUs-6HRsgsrNG-!;(kcc=(0ME z$mS!3BBwsb7zmhWT$MMC$sWTkX=66&+ErChD)@@)XuU~!LqufQSgFqC%t*O~{PlQt zs*z`jCT&bX!o&$Z3~dX?*f^6!e@^I)G}LO&Y6CvxEat>%>Q^UH5=N@MS7{QWTY67B?)W!Z2J{r2mq6nlWaXhaKhU+}FDX7eugcBPIvWB9fIRcli zO?{by_krXqevKLh`+m%QV2(T=`ug)*>-KM^>EDp^Ro`s!V@#WJ>PS>;`r+g6L}YtT z#Ko~v@HJ^RGw%YnGSg#-)ON9SAwG^RLf*=m(8!~#3fbes*&NW@^qV_+4r5`5v!e1y zbr0P~TXi+UrcR@F-aL(h<3O(q!fGYO43qwI*WQUqioS9}pY*7g-QX;38B>i;W10!0 zSk$}+Z*+DRJQWgM6)RGr;p1wV!H!3I;~*xi{lK#b@6;J;m1t#mE@I^zH~L^*>LY>C zW=qkRkoeZb&G6X0%$Z;m%@K{mDv2WEX-?;JlB>UY68CX?b<(y$dG)%d1IF)Rt&Z83 z6M$#{ODZ3k{dZmP007xD@Y%@JiZi69>7~>sw#amvvhzbm(kWO`d~SJIo6ooA%)9>P z5b_Lv4|3hmq;E#^gvXFKWPHSRxe;P7)?&yICM*K zwyT44mk-gVwnHH>SO*hl;ZSqHqooIB&O{;wO$zyml{bKPdR+xEhdvG6_ipxbdy=0 zP1;{KF((Fga%CK1G_deJfl1TJ>uJ3p+acSf2nFVZrX)hbNnBI~bKDbJam%NMM~|A) z0Q!zA6`szR$Hk}uJhR9uev%arnT3@`R$((+uDQ&an&G{DRaaxxd z>Ui#=$S1`a5l+u#AS z$zoli&rp2ypri;UvLU5mb_(JTW`=J|;jUcQZX{{=(eVKuWulqRCWI0T#~+9cF9h@h zz$iH?6rr`opDX1qq!MgB_v+toU4JK((ntAXD6HzB8v zWE(X-xw>;*i;O~@yk#m}d3sLPA$~hv9n23-Bh5>i_X);a=ObyTb5$Ax`{SAAKw$!) z&6gA`aF7V3z}f(?)U+O3AgyP*2F%KynJNS4Py^j0FXAs|$hrcdn>e!c6*(awGynp@bi!XQkVypF-4cc46PX}6do z!(ov!_Rp3JL`15(BO~XM=xR<_br-8LRfp|;)#~$bt=bFE^w{fqw#Z_*KHTQ$Xh$hx_X_l#F2bO7{1rt4cL|3cZ4JPb>M%V) z27*}hAI{{c?vst!N#9fiwfMk6SQ8r&>R!&2$9@7befeg)*T*2@9-p6|AJOVkE5+F0Mqy~Dr8v83z79dTym6p>*KKygV-L`!?xX?`-3nqpnRDavak;h(fR zR&`AE(5BgeC!}9P<{l*wuywXCU>WeYK=$o8D*{IXvSKu37D@cb(jI{75gLFGR z*Zk?KhR#lUd&NK%`;uEN!cpwKbc{|2Z7g>RWCsfkq+$SV)mALUvv55AVtlH7B`{u{ z5V0;eVkzXC^jtiCXy3x?y+Zv}S8ITT=12&jUm5kf&@tEHXl#MI3A{L0l+MOFgrw(( z&mj9p_haj9xqnqm*LgouwsO8@4hT!cuNlRwj$hX^ZA%=$N1uKXEnP-i=_EulNgH{S z4fkZ!;~!C-X}=wVDE9+F0?yaNF6+|IfbE6Ms)?02wef6oE<&#pe{hthY{$7q{sx$F z{2y2+`-7{e1f&t_;B+W1>@?Fzrf`@Kb(5B~e$QRJ%`2MY5xd#)dh;+1a|~m6olH&i zu0t1e^F-f*{{>Gx@Z2#ghsM2j zwJ5zYzFWFVH2FVOXEAE;;}ihm&GUPP3vgUeSG{j`wbqNx%Q|)1GhsU4^kt`~p^~4g zfs>?EJ6$QKCRSwlRYz2SH`+%4yC8Z_iQ^as{pFI{NP{|>QA4K)2(D~- z-2Aqyh|Fj3>pYUUexM}^cOFODE-?Rueoeuy>u7vO9`yVRGys$Vf9uN(eiI|+lJjJo&n0 z(g7oegPwE*7STxks{_9fv>rgJB|-;>gciwCJ%tD!vhvFF7r31yt*T~Oc05&!FDr?* zzxf68WwRfe=G<|nU*I0eX^)wnp5{bE;K`nY&Qw7G=}4TFZ{9flbH14wK8_t-zf5x8 zf(=ModBw9ecZ9#fnSyx+24YBaY0W$NK_L0{6VK1_FPYr!heju}WWAY0$;)e+SU)#~ zB@PP1nkoU(Al7k!d=Qy91c{DoXdrIRzl=^-uU>a-MWxv(ONn4c11dL~WJQ(QQ}W@xv#xO{GZ_ubn91&Cfg zD|sY#p_VmB_mctLHHL)_!<-FT_`vRLFiKKjN(PIItpD|8{r-kT)ykXmfJ_cgM{J&( z!>XRizD*t~Bt{v50ZVgrMiJ*aY9y8z)f$AIzueEizg|6iecrJS546DPiI{=?F9S19 z^a3gf0d0%pV9b!TSR9Ux{NL;2Z*B?M!KWMbbd(x*7eagX3!$mrQYtQ6)KVUGg6LYFcO z5toB)b+=lwvY@DOHvmu&@g&Opk0%H-p7w-;P!MrYSaq9FBh3y<5~q+laJ0ZP!Ab{O z1RMl96gm{1@sIl;7bSg6mG=mIv1!1{GbdwNc_tx(7Ax~Hl+SJD!Z_zJ1Z=6+IFg5* z2k{>}`djy``es%FgNg~+EL_V;K+r2Fggl560-N5U8!vRmtK#rF0I&V?GfS`;S%c;i zIdBMIr#M}eLSSykLtc$+*<@1#CL+OdCZFU=W^qHt Date: Sat, 26 Oct 2019 01:38:02 +0530 Subject: [PATCH 4/9] Update GAN.py --- neural_network/GAN.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/neural_network/GAN.py b/neural_network/GAN.py index 29a88e22638f..e74884a60f32 100644 --- a/neural_network/GAN.py +++ b/neural_network/GAN.py @@ -1,9 +1,7 @@ -import numpy as np,sys,time +import numpy as np from sklearn.utils import shuffle import matplotlib.pyplot as plt -import matplotlib from tensorflow.examples.tutorials.mnist import input_data -import os import numpy as np,sys from sklearn.utils import shuffle import matplotlib.pyplot as plt From 12c767494c8bc24d429582c152228c10f561611b Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Sat, 26 Oct 2019 01:42:09 +0530 Subject: [PATCH 5/9] Update GAN.py --- neural_network/GAN.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/neural_network/GAN.py b/neural_network/GAN.py index e74884a60f32..b1299990dc3d 100644 --- a/neural_network/GAN.py +++ b/neural_network/GAN.py @@ -1,9 +1,6 @@ import numpy as np from sklearn.utils import shuffle -import matplotlib.pyplot as plt from tensorflow.examples.tutorials.mnist import input_data -import numpy as np,sys -from sklearn.utils import shuffle import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec From f02d7c55b73e6708a7308e07ecb74e5059e27c29 Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Sat, 26 Oct 2019 01:56:19 +0530 Subject: [PATCH 6/9] Delete GAN.py --- neural_network/GAN.py | 392 ------------------------------------------ 1 file changed, 392 deletions(-) delete mode 100644 neural_network/GAN.py diff --git a/neural_network/GAN.py b/neural_network/GAN.py deleted file mode 100644 index b1299990dc3d..000000000000 --- a/neural_network/GAN.py +++ /dev/null @@ -1,392 +0,0 @@ -import numpy as np -from sklearn.utils import shuffle -from tensorflow.examples.tutorials.mnist import input_data -import matplotlib.pyplot as plt -import matplotlib.gridspec as gridspec - -random_numer = 42 - - -np.random.seed(random_numer) -def ReLu(x): - mask = (x>0) * 1.0 - return mask *x -def d_ReLu(x): - mask = (x>0) * 1.0 - return mask - -def arctan(x): - return np.arctan(x) -def d_arctan(x): - return 1 / (1 + x ** 2) - -def log(x): - return 1 / ( 1+ np.exp(-1*x)) -def d_log(x): - return log(x) * (1 - log(x)) - -def tanh(x): - return np.tanh(x) -def d_tanh(x): - return 1 - np.tanh(x) ** 2 - -def plot(samples): - fig = plt.figure(figsize=(4, 4)) - gs = gridspec.GridSpec(4, 4) - gs.update(wspace=0.05, hspace=0.05) - - for i, sample in enumerate(samples): - ax = plt.subplot(gs[i]) - plt.axis('off') - ax.set_xticklabels([]) - ax.set_yticklabels([]) - ax.set_aspect('equal') - plt.imshow(sample.reshape(28, 28), cmap='Greys_r') - - return fig - - - -# 1. Load Data and declare hyper -print('--------- Load Data ----------') -mnist = input_data.read_data_sets('MNIST_data', one_hot=False) -temp = mnist.test -images, labels = temp.images, temp.labels -images, labels = shuffle(np.asarray(images),np.asarray(labels)) -num_epoch = 10 -learing_rate = 0.00009 -G_input = 100 -hidden_input,hidden_input2,hidden_input3 = 128,256,346 -hidden_input4,hidden_input5,hidden_input6 = 480,560,686 - - - -print('--------- Declare Hyper Parameters ----------') -# 2. Declare Weights -D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.))) *0.002 -# D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -D_b1 = np.zeros(hidden_input) - -D_W2 = np.random.normal(size=(hidden_input,1),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 -# D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.))) *0.002 -D_b2 = np.zeros(1) - - -G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b1 = np.zeros(hidden_input) - -G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b2 = np.zeros(hidden_input2) - -G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b3 = np.zeros(hidden_input3) - -G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b4 = np.zeros(hidden_input4) - -G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b5 = np.zeros(hidden_input5) - -G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.))) *0.002 -# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 -G_b6 = np.zeros(hidden_input6) - -G_W7 = np.random.normal(size=(hidden_input6,784),scale=(1. / np.sqrt(hidden_input6 / 2.))) *0.002 -# G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.))) *0.002 -G_b7 = np.zeros(784) - -# 3. For Adam Optimzier -v1,m1 = 0,0 -v2,m2 = 0,0 -v3,m3 = 0,0 -v4,m4 = 0,0 - -v5,m5 = 0,0 -v6,m6 = 0,0 -v7,m7 = 0,0 -v8,m8 = 0,0 -v9,m9 = 0,0 -v10,m10 = 0,0 -v11,m11 = 0,0 -v12,m12 = 0,0 - -v13,m13 = 0,0 -v14,m14 = 0,0 - -v15,m15 = 0,0 -v16,m16 = 0,0 - -v17,m17 = 0,0 -v18,m18 = 0,0 - - -beta_1,beta_2,eps = 0.9,0.999,0.00000001 - -print('--------- Started Training ----------') -for iter in range(num_epoch): - - random_int = np.random.randint(len(images) - 5) - current_image = np.expand_dims(images[random_int],axis=0) - - # Func: Generate The first Fake Data - Z = np.random.uniform(-1., 1., size=[1, G_input]) - Gl1 = Z.dot(G_W1) + G_b1 - Gl1A = arctan(Gl1) - Gl2 = Gl1A.dot(G_W2) + G_b2 - Gl2A = ReLu(Gl2) - Gl3 = Gl2A.dot(G_W3) + G_b3 - Gl3A = arctan(Gl3) - - Gl4 = Gl3A.dot(G_W4) + G_b4 - Gl4A = ReLu(Gl4) - Gl5 = Gl4A.dot(G_W5) + G_b5 - Gl5A = tanh(Gl5) - Gl6 = Gl5A.dot(G_W6) + G_b6 - Gl6A = ReLu(Gl6) - Gl7 = Gl6A.dot(G_W7) + G_b7 - - current_fake_data = log(Gl7) - - # Func: Forward Feed for Real data - Dl1_r = current_image.dot(D_W1) + D_b1 - Dl1_rA = ReLu(Dl1_r) - Dl2_r = Dl1_rA.dot(D_W2) + D_b2 - Dl2_rA = log(Dl2_r) - - # Func: Forward Feed for Fake Data - Dl1_f = current_fake_data.dot(D_W1) + D_b1 - Dl1_fA = ReLu(Dl1_f) - Dl2_f = Dl1_fA.dot(D_W2) + D_b2 - Dl2_fA = log(Dl2_f) - - # Func: Cost D - D_cost = -np.log(Dl2_rA) + np.log(1.0- Dl2_fA) - - # Func: Gradient - grad_f_w2_part_1 = 1/(1.0- Dl2_fA) - grad_f_w2_part_2 = d_log(Dl2_f) - grad_f_w2_part_3 = Dl1_fA - grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2) - grad_f_b2 = grad_f_w2_part_1 * grad_f_w2_part_2 - - grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T) - grad_f_w1_part_2 = d_ReLu(Dl1_f) - grad_f_w1_part_3 = current_fake_data - grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2) - grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2 - - grad_r_w2_part_1 = - 1/Dl2_rA - grad_r_w2_part_2 = d_log(Dl2_r) - grad_r_w2_part_3 = Dl1_rA - grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2) - grad_r_b2 = grad_r_w2_part_1 * grad_r_w2_part_2 - - grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T) - grad_r_w1_part_2 = d_ReLu(Dl1_r) - grad_r_w1_part_3 = current_image - grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2) - grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2 - - grad_w1 =grad_f_w1 + grad_r_w1 - grad_b1 =grad_f_b1 + grad_r_b1 - - grad_w2 =grad_f_w2 + grad_r_w2 - grad_b2 =grad_f_b2 + grad_r_b2 - - # ---- Update Gradient ---- - m1 = beta_1 * m1 + (1 - beta_1) * grad_w1 - v1 = beta_2 * v1 + (1 - beta_2) * grad_w1 ** 2 - - m2 = beta_1 * m2 + (1 - beta_1) * grad_b1 - v2 = beta_2 * v2 + (1 - beta_2) * grad_b1 ** 2 - - m3 = beta_1 * m3 + (1 - beta_1) * grad_w2 - v3 = beta_2 * v3 + (1 - beta_2) * grad_w2 ** 2 - - m4 = beta_1 * m4 + (1 - beta_1) * grad_b2 - v4 = beta_2 * v4 + (1 - beta_2) * grad_b2 ** 2 - - D_W1 = D_W1 - (learing_rate / (np.sqrt(v1 /(1-beta_2) ) + eps)) * (m1/(1-beta_1)) - D_b1 = D_b1 - (learing_rate / (np.sqrt(v2 /(1-beta_2) ) + eps)) * (m2/(1-beta_1)) - - D_W2 = D_W2 - (learing_rate / (np.sqrt(v3 /(1-beta_2) ) + eps)) * (m3/(1-beta_1)) - D_b2 = D_b2 - (learing_rate / (np.sqrt(v4 /(1-beta_2) ) + eps)) * (m4/(1-beta_1)) - - # Func: Forward Feed for G - Z = np.random.uniform(-1., 1., size=[1, G_input]) - Gl1 = Z.dot(G_W1) + G_b1 - Gl1A = arctan(Gl1) - Gl2 = Gl1A.dot(G_W2) + G_b2 - Gl2A = ReLu(Gl2) - Gl3 = Gl2A.dot(G_W3) + G_b3 - Gl3A = arctan(Gl3) - - Gl4 = Gl3A.dot(G_W4) + G_b4 - Gl4A = ReLu(Gl4) - Gl5 = Gl4A.dot(G_W5) + G_b5 - Gl5A = tanh(Gl5) - Gl6 = Gl5A.dot(G_W6) + G_b6 - Gl6A = ReLu(Gl6) - Gl7 = Gl6A.dot(G_W7) + G_b7 - - current_fake_data = log(Gl7) - - Dl1 = current_fake_data.dot(D_W1) + D_b1 - Dl1_A = ReLu(Dl1) - Dl2 = Dl1_A.dot(D_W2) + D_b2 - Dl2_A = log(Dl2) - - # Func: Cost G - G_cost = -np.log(Dl2_A) - - # Func: Gradient - grad_G_w7_part_1 = ((-1/Dl2_A) * d_log(Dl2).dot(D_W2.T) * (d_ReLu(Dl1))).dot(D_W1.T) - grad_G_w7_part_2 = d_log(Gl7) - grad_G_w7_part_3 = Gl6A - grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_1) - grad_G_b7 = grad_G_w7_part_1 * grad_G_w7_part_2 - - grad_G_w6_part_1 = (grad_G_w7_part_1 * grad_G_w7_part_2).dot(G_W7.T) - grad_G_w6_part_2 = d_ReLu(Gl6) - grad_G_w6_part_3 = Gl5A - grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2) - grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2) - - grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T) - grad_G_w5_part_2 = d_tanh(Gl5) - grad_G_w5_part_3 = Gl4A - grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2) - grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2) - - grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T) - grad_G_w4_part_2 = d_ReLu(Gl4) - grad_G_w4_part_3 = Gl3A - grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2) - grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2) - - grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T) - grad_G_w3_part_2 = d_arctan(Gl3) - grad_G_w3_part_3 = Gl2A - grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2) - grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2) - - grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T) - grad_G_w2_part_2 = d_ReLu(Gl2) - grad_G_w2_part_3 = Gl1A - grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2) - grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2) - - grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T) - grad_G_w1_part_2 = d_arctan(Gl1) - grad_G_w1_part_3 = Z - grad_G_w1 = grad_G_w1_part_3.T.dot(grad_G_w1_part_1 * grad_G_w1_part_2) - grad_G_b1 = grad_G_w1_part_1 * grad_G_w1_part_2 - - # ---- Update Gradient ---- - m5 = beta_1 * m5 + (1 - beta_1) * grad_G_w1 - v5 = beta_2 * v5 + (1 - beta_2) * grad_G_w1 ** 2 - - m6 = beta_1 * m6 + (1 - beta_1) * grad_G_b1 - v6 = beta_2 * v6 + (1 - beta_2) * grad_G_b1 ** 2 - - m7 = beta_1 * m7 + (1 - beta_1) * grad_G_w2 - v7 = beta_2 * v7 + (1 - beta_2) * grad_G_w2 ** 2 - - m8 = beta_1 * m8 + (1 - beta_1) * grad_G_b2 - v8 = beta_2 * v8 + (1 - beta_2) * grad_G_b2 ** 2 - - m9 = beta_1 * m9 + (1 - beta_1) * grad_G_w3 - v9 = beta_2 * v9 + (1 - beta_2) * grad_G_w3 ** 2 - - m10 = beta_1 * m10 + (1 - beta_1) * grad_G_b3 - v10 = beta_2 * v10 + (1 - beta_2) * grad_G_b3 ** 2 - - m11 = beta_1 * m11 + (1 - beta_1) * grad_G_w4 - v11 = beta_2 * v11 + (1 - beta_2) * grad_G_w4 ** 2 - - m12 = beta_1 * m12 + (1 - beta_1) * grad_G_b4 - v12 = beta_2 * v12 + (1 - beta_2) * grad_G_b4 ** 2 - - m13 = beta_1 * m13 + (1 - beta_1) * grad_G_w5 - v13 = beta_2 * v13 + (1 - beta_2) * grad_G_w5 ** 2 - - m14 = beta_1 * m14 + (1 - beta_1) * grad_G_b5 - v14 = beta_2 * v14 + (1 - beta_2) * grad_G_b5 ** 2 - - m15 = beta_1 * m15 + (1 - beta_1) * grad_G_w6 - v15 = beta_2 * v15 + (1 - beta_2) * grad_G_w6 ** 2 - - m16 = beta_1 * m16 + (1 - beta_1) * grad_G_b6 - v16 = beta_2 * v16 + (1 - beta_2) * grad_G_b6 ** 2 - - m17 = beta_1 * m17 + (1 - beta_1) * grad_G_w7 - v17 = beta_2 * v17 + (1 - beta_2) * grad_G_w7 ** 2 - - m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7 - v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2 - - G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2) ) + eps)) * (m5/(1-beta_1)) - G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2) ) + eps)) * (m6/(1-beta_1)) - - G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2) ) + eps)) * (m7/(1-beta_1)) - G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2) ) + eps)) * (m8/(1-beta_1)) - - G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2) ) + eps)) * (m9/(1-beta_1)) - G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2) ) + eps)) * (m10/(1-beta_1)) - - G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2) ) + eps)) * (m11/(1-beta_1)) - G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2) ) + eps)) * (m12/(1-beta_1)) - - G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2) ) + eps)) * (m13/(1-beta_1)) - G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2) ) + eps)) * (m14/(1-beta_1)) - - G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2) ) + eps)) * (m15/(1-beta_1)) - G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2) ) + eps)) * (m16/(1-beta_1)) - - G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2) ) + eps)) * (m17/(1-beta_1)) - G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2) ) + eps)) * (m18/(1-beta_1)) - - # --- Print Error ---- - #print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') - - if iter == 0: - learing_rate = learing_rate * 0.01 - if iter == 40: - learing_rate = learing_rate * 0.01 - - # ---- Print to Out put ---- - if iter%10 == 0: - - print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') - print('--------- Show Example Result See Tab Above ----------') - print('--------- Wait for the image to load ---------') - Z = np.random.uniform(-1., 1., size=[16, G_input]) - - Gl1 = Z.dot(G_W1) + G_b1 - Gl1A = arctan(Gl1) - Gl2 = Gl1A.dot(G_W2) + G_b2 - Gl2A = ReLu(Gl2) - Gl3 = Gl2A.dot(G_W3) + G_b3 - Gl3A = arctan(Gl3) - - Gl4 = Gl3A.dot(G_W4) + G_b4 - Gl4A = ReLu(Gl4) - Gl5 = Gl4A.dot(G_W5) + G_b5 - Gl5A = tanh(Gl5) - Gl6 = Gl5A.dot(G_W6) + G_b6 - Gl6A = ReLu(Gl6) - Gl7 = Gl6A.dot(G_W7) + G_b7 - - current_fake_data = log(Gl7) - - fig = plot(current_fake_data) - fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+"_Ginput_"+str(G_input)+ \ - "_hiddenone"+str(hidden_input) + "_hiddentwo"+str(hidden_input2) + "_LR_" + str(learing_rate) - ), bbox_inches='tight') -#for complete explanation visit https://towardsdatascience.com/only-numpy-implementing-gan-general-adversarial-networks-and-adam-optimizer-using-numpy-with-2a7e4e032021 -# -- end code -- From 393d1b6e781291b36510ef72873051d532e8ad54 Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Sat, 26 Oct 2019 01:56:47 +0530 Subject: [PATCH 7/9] Create gan.py --- neural_network/gan.py | 391 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 391 insertions(+) create mode 100644 neural_network/gan.py diff --git a/neural_network/gan.py b/neural_network/gan.py new file mode 100644 index 000000000000..85dcfa74b890 --- /dev/null +++ b/neural_network/gan.py @@ -0,0 +1,391 @@ +import numpy as np +from sklearn.utils import shuffle +from tensorflow.examples.tutorials.mnist import input_data +import matplotlib.pyplot as plt +import matplotlib.gridspec as gridspec + +random_numer = 42 + +np.random.seed(random_numer) +def ReLu(x): + mask = (x>0) * 1.0 + return mask *x +def d_ReLu(x): + mask = (x>0) * 1.0 + return mask + +def arctan(x): + return np.arctan(x) +def d_arctan(x): + return 1 / (1 + x ** 2) + +def log(x): + return 1 / ( 1+ np.exp(-1*x)) +def d_log(x): + return log(x) * (1 - log(x)) + +def tanh(x): + return np.tanh(x) +def d_tanh(x): + return 1 - np.tanh(x) ** 2 + +def plot(samples): + fig = plt.figure(figsize=(4, 4)) + gs = gridspec.GridSpec(4, 4) + gs.update(wspace=0.05, hspace=0.05) + + for i, sample in enumerate(samples): + ax = plt.subplot(gs[i]) + plt.axis('off') + ax.set_xticklabels([]) + ax.set_yticklabels([]) + ax.set_aspect('equal') + plt.imshow(sample.reshape(28, 28), cmap='Greys_r') + + return fig + + + +# 1. Load Data and declare hyper +print('--------- Load Data ----------') +mnist = input_data.read_data_sets('MNIST_data', one_hot=False) +temp = mnist.test +images, labels = temp.images, temp.labels +images, labels = shuffle(np.asarray(images),np.asarray(labels)) +num_epoch = 10 +learing_rate = 0.00009 +G_input = 100 +hidden_input,hidden_input2,hidden_input3 = 128,256,346 +hidden_input4,hidden_input5,hidden_input6 = 480,560,686 + + + +print('--------- Declare Hyper Parameters ----------') +# 2. Declare Weights +D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.))) *0.002 +# D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +D_b1 = np.zeros(hidden_input) + +D_W2 = np.random.normal(size=(hidden_input,1),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +# D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.))) *0.002 +D_b2 = np.zeros(1) + + +G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b1 = np.zeros(hidden_input) + +G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b2 = np.zeros(hidden_input2) + +G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b3 = np.zeros(hidden_input3) + +G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b4 = np.zeros(hidden_input4) + +G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b5 = np.zeros(hidden_input5) + +G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.))) *0.002 +# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.))) *0.002 +G_b6 = np.zeros(hidden_input6) + +G_W7 = np.random.normal(size=(hidden_input6,784),scale=(1. / np.sqrt(hidden_input6 / 2.))) *0.002 +# G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.))) *0.002 +G_b7 = np.zeros(784) + +# 3. For Adam Optimzier +v1,m1 = 0,0 +v2,m2 = 0,0 +v3,m3 = 0,0 +v4,m4 = 0,0 + +v5,m5 = 0,0 +v6,m6 = 0,0 +v7,m7 = 0,0 +v8,m8 = 0,0 +v9,m9 = 0,0 +v10,m10 = 0,0 +v11,m11 = 0,0 +v12,m12 = 0,0 + +v13,m13 = 0,0 +v14,m14 = 0,0 + +v15,m15 = 0,0 +v16,m16 = 0,0 + +v17,m17 = 0,0 +v18,m18 = 0,0 + + +beta_1,beta_2,eps = 0.9,0.999,0.00000001 + +print('--------- Started Training ----------') +for iter in range(num_epoch): + + random_int = np.random.randint(len(images) - 5) + current_image = np.expand_dims(images[random_int],axis=0) + + # Func: Generate The first Fake Data + Z = np.random.uniform(-1., 1., size=[1, G_input]) + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + # Func: Forward Feed for Real data + Dl1_r = current_image.dot(D_W1) + D_b1 + Dl1_rA = ReLu(Dl1_r) + Dl2_r = Dl1_rA.dot(D_W2) + D_b2 + Dl2_rA = log(Dl2_r) + + # Func: Forward Feed for Fake Data + Dl1_f = current_fake_data.dot(D_W1) + D_b1 + Dl1_fA = ReLu(Dl1_f) + Dl2_f = Dl1_fA.dot(D_W2) + D_b2 + Dl2_fA = log(Dl2_f) + + # Func: Cost D + D_cost = -np.log(Dl2_rA) + np.log(1.0- Dl2_fA) + + # Func: Gradient + grad_f_w2_part_1 = 1/(1.0- Dl2_fA) + grad_f_w2_part_2 = d_log(Dl2_f) + grad_f_w2_part_3 = Dl1_fA + grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2) + grad_f_b2 = grad_f_w2_part_1 * grad_f_w2_part_2 + + grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T) + grad_f_w1_part_2 = d_ReLu(Dl1_f) + grad_f_w1_part_3 = current_fake_data + grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2) + grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2 + + grad_r_w2_part_1 = - 1/Dl2_rA + grad_r_w2_part_2 = d_log(Dl2_r) + grad_r_w2_part_3 = Dl1_rA + grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2) + grad_r_b2 = grad_r_w2_part_1 * grad_r_w2_part_2 + + grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T) + grad_r_w1_part_2 = d_ReLu(Dl1_r) + grad_r_w1_part_3 = current_image + grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2) + grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2 + + grad_w1 =grad_f_w1 + grad_r_w1 + grad_b1 =grad_f_b1 + grad_r_b1 + + grad_w2 =grad_f_w2 + grad_r_w2 + grad_b2 =grad_f_b2 + grad_r_b2 + + # ---- Update Gradient ---- + m1 = beta_1 * m1 + (1 - beta_1) * grad_w1 + v1 = beta_2 * v1 + (1 - beta_2) * grad_w1 ** 2 + + m2 = beta_1 * m2 + (1 - beta_1) * grad_b1 + v2 = beta_2 * v2 + (1 - beta_2) * grad_b1 ** 2 + + m3 = beta_1 * m3 + (1 - beta_1) * grad_w2 + v3 = beta_2 * v3 + (1 - beta_2) * grad_w2 ** 2 + + m4 = beta_1 * m4 + (1 - beta_1) * grad_b2 + v4 = beta_2 * v4 + (1 - beta_2) * grad_b2 ** 2 + + D_W1 = D_W1 - (learing_rate / (np.sqrt(v1 /(1-beta_2) ) + eps)) * (m1/(1-beta_1)) + D_b1 = D_b1 - (learing_rate / (np.sqrt(v2 /(1-beta_2) ) + eps)) * (m2/(1-beta_1)) + + D_W2 = D_W2 - (learing_rate / (np.sqrt(v3 /(1-beta_2) ) + eps)) * (m3/(1-beta_1)) + D_b2 = D_b2 - (learing_rate / (np.sqrt(v4 /(1-beta_2) ) + eps)) * (m4/(1-beta_1)) + + # Func: Forward Feed for G + Z = np.random.uniform(-1., 1., size=[1, G_input]) + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + Dl1 = current_fake_data.dot(D_W1) + D_b1 + Dl1_A = ReLu(Dl1) + Dl2 = Dl1_A.dot(D_W2) + D_b2 + Dl2_A = log(Dl2) + + # Func: Cost G + G_cost = -np.log(Dl2_A) + + # Func: Gradient + grad_G_w7_part_1 = ((-1/Dl2_A) * d_log(Dl2).dot(D_W2.T) * (d_ReLu(Dl1))).dot(D_W1.T) + grad_G_w7_part_2 = d_log(Gl7) + grad_G_w7_part_3 = Gl6A + grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_1) + grad_G_b7 = grad_G_w7_part_1 * grad_G_w7_part_2 + + grad_G_w6_part_1 = (grad_G_w7_part_1 * grad_G_w7_part_2).dot(G_W7.T) + grad_G_w6_part_2 = d_ReLu(Gl6) + grad_G_w6_part_3 = Gl5A + grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2) + grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2) + + grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T) + grad_G_w5_part_2 = d_tanh(Gl5) + grad_G_w5_part_3 = Gl4A + grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2) + grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2) + + grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T) + grad_G_w4_part_2 = d_ReLu(Gl4) + grad_G_w4_part_3 = Gl3A + grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2) + grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2) + + grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T) + grad_G_w3_part_2 = d_arctan(Gl3) + grad_G_w3_part_3 = Gl2A + grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2) + grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2) + + grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T) + grad_G_w2_part_2 = d_ReLu(Gl2) + grad_G_w2_part_3 = Gl1A + grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2) + grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2) + + grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T) + grad_G_w1_part_2 = d_arctan(Gl1) + grad_G_w1_part_3 = Z + grad_G_w1 = grad_G_w1_part_3.T.dot(grad_G_w1_part_1 * grad_G_w1_part_2) + grad_G_b1 = grad_G_w1_part_1 * grad_G_w1_part_2 + + # ---- Update Gradient ---- + m5 = beta_1 * m5 + (1 - beta_1) * grad_G_w1 + v5 = beta_2 * v5 + (1 - beta_2) * grad_G_w1 ** 2 + + m6 = beta_1 * m6 + (1 - beta_1) * grad_G_b1 + v6 = beta_2 * v6 + (1 - beta_2) * grad_G_b1 ** 2 + + m7 = beta_1 * m7 + (1 - beta_1) * grad_G_w2 + v7 = beta_2 * v7 + (1 - beta_2) * grad_G_w2 ** 2 + + m8 = beta_1 * m8 + (1 - beta_1) * grad_G_b2 + v8 = beta_2 * v8 + (1 - beta_2) * grad_G_b2 ** 2 + + m9 = beta_1 * m9 + (1 - beta_1) * grad_G_w3 + v9 = beta_2 * v9 + (1 - beta_2) * grad_G_w3 ** 2 + + m10 = beta_1 * m10 + (1 - beta_1) * grad_G_b3 + v10 = beta_2 * v10 + (1 - beta_2) * grad_G_b3 ** 2 + + m11 = beta_1 * m11 + (1 - beta_1) * grad_G_w4 + v11 = beta_2 * v11 + (1 - beta_2) * grad_G_w4 ** 2 + + m12 = beta_1 * m12 + (1 - beta_1) * grad_G_b4 + v12 = beta_2 * v12 + (1 - beta_2) * grad_G_b4 ** 2 + + m13 = beta_1 * m13 + (1 - beta_1) * grad_G_w5 + v13 = beta_2 * v13 + (1 - beta_2) * grad_G_w5 ** 2 + + m14 = beta_1 * m14 + (1 - beta_1) * grad_G_b5 + v14 = beta_2 * v14 + (1 - beta_2) * grad_G_b5 ** 2 + + m15 = beta_1 * m15 + (1 - beta_1) * grad_G_w6 + v15 = beta_2 * v15 + (1 - beta_2) * grad_G_w6 ** 2 + + m16 = beta_1 * m16 + (1 - beta_1) * grad_G_b6 + v16 = beta_2 * v16 + (1 - beta_2) * grad_G_b6 ** 2 + + m17 = beta_1 * m17 + (1 - beta_1) * grad_G_w7 + v17 = beta_2 * v17 + (1 - beta_2) * grad_G_w7 ** 2 + + m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7 + v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2 + + G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2) ) + eps)) * (m5/(1-beta_1)) + G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2) ) + eps)) * (m6/(1-beta_1)) + + G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2) ) + eps)) * (m7/(1-beta_1)) + G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2) ) + eps)) * (m8/(1-beta_1)) + + G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2) ) + eps)) * (m9/(1-beta_1)) + G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2) ) + eps)) * (m10/(1-beta_1)) + + G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2) ) + eps)) * (m11/(1-beta_1)) + G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2) ) + eps)) * (m12/(1-beta_1)) + + G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2) ) + eps)) * (m13/(1-beta_1)) + G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2) ) + eps)) * (m14/(1-beta_1)) + + G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2) ) + eps)) * (m15/(1-beta_1)) + G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2) ) + eps)) * (m16/(1-beta_1)) + + G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2) ) + eps)) * (m17/(1-beta_1)) + G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2) ) + eps)) * (m18/(1-beta_1)) + + # --- Print Error ---- + #print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') + + if iter == 0: + learing_rate = learing_rate * 0.01 + if iter == 40: + learing_rate = learing_rate * 0.01 + + # ---- Print to Out put ---- + if iter%10 == 0: + + print("Current Iter: ",iter, " Current D cost:",D_cost, " Current G cost: ", G_cost,end='\r') + print('--------- Show Example Result See Tab Above ----------') + print('--------- Wait for the image to load ---------') + Z = np.random.uniform(-1., 1., size=[16, G_input]) + + Gl1 = Z.dot(G_W1) + G_b1 + Gl1A = arctan(Gl1) + Gl2 = Gl1A.dot(G_W2) + G_b2 + Gl2A = ReLu(Gl2) + Gl3 = Gl2A.dot(G_W3) + G_b3 + Gl3A = arctan(Gl3) + + Gl4 = Gl3A.dot(G_W4) + G_b4 + Gl4A = ReLu(Gl4) + Gl5 = Gl4A.dot(G_W5) + G_b5 + Gl5A = tanh(Gl5) + Gl6 = Gl5A.dot(G_W6) + G_b6 + Gl6A = ReLu(Gl6) + Gl7 = Gl6A.dot(G_W7) + G_b7 + + current_fake_data = log(Gl7) + + fig = plot(current_fake_data) + fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+"_Ginput_"+str(G_input)+ \ + "_hiddenone"+str(hidden_input) + "_hiddentwo"+str(hidden_input2) + "_LR_" + str(learing_rate) + ), bbox_inches='tight') +#for complete explanation visit https://towardsdatascience.com/only-numpy-implementing-gan-general-adversarial-networks-and-adam-optimizer-using-numpy-with-2a7e4e032021 +# -- end code -- From 6ec69ddaf60cb53dd8b8342b8c5bfca020143df7 Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Sat, 26 Oct 2019 02:17:41 +0530 Subject: [PATCH 8/9] Update gan.py --- neural_network/gan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neural_network/gan.py b/neural_network/gan.py index 85dcfa74b890..b7aa408a4048 100644 --- a/neural_network/gan.py +++ b/neural_network/gan.py @@ -1,8 +1,8 @@ +import matplotlib.gridspec as gridspec +import matplotlib.pyplot as plt import numpy as np from sklearn.utils import shuffle from tensorflow.examples.tutorials.mnist import input_data -import matplotlib.pyplot as plt -import matplotlib.gridspec as gridspec random_numer = 42 From 86f3b0d7aa9cc5411ddbb57c2efcae0dfc9e84e7 Mon Sep 17 00:00:00 2001 From: DevilLord9967 Date: Sat, 26 Oct 2019 14:27:08 +0530 Subject: [PATCH 9/9] input_data import file --- neural_network/gan.py | 2 +- neural_network/input_data.py | 332 +++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 neural_network/input_data.py diff --git a/neural_network/gan.py b/neural_network/gan.py index b7aa408a4048..edfff420547b 100644 --- a/neural_network/gan.py +++ b/neural_network/gan.py @@ -2,7 +2,7 @@ import matplotlib.pyplot as plt import numpy as np from sklearn.utils import shuffle -from tensorflow.examples.tutorials.mnist import input_data +import input_data random_numer = 42 diff --git a/neural_network/input_data.py b/neural_network/input_data.py new file mode 100644 index 000000000000..983063f0b72d --- /dev/null +++ b/neural_network/input_data.py @@ -0,0 +1,332 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions for downloading and reading MNIST data (deprecated). + +This module and all its submodules are deprecated. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import gzip +import os + +import numpy +from six.moves import urllib +from six.moves import xrange # pylint: disable=redefined-builtin + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import random_seed +from tensorflow.python.platform import gfile +from tensorflow.python.util.deprecation import deprecated + +_Datasets = collections.namedtuple('_Datasets', ['train', 'validation', 'test']) + +# CVDF mirror of http://yann.lecun.com/exdb/mnist/ +DEFAULT_SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + + +def _read32(bytestream): + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(4), dtype=dt)[0] + + +@deprecated(None, 'Please use tf.data to implement this functionality.') +def _extract_images(f): + """Extract the images into a 4D uint8 numpy array [index, y, x, depth]. + + Args: + f: A file object that can be passed into a gzip reader. + + Returns: + data: A 4D uint8 numpy array [index, y, x, depth]. + + Raises: + ValueError: If the bytestream does not start with 2051. + + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError('Invalid magic number %d in MNIST image file: %s' % + (magic, f.name)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + + +@deprecated(None, 'Please use tf.one_hot on tensors.') +def _dense_to_one_hot(labels_dense, num_classes): + """Convert class labels from scalars to one-hot vectors.""" + num_labels = labels_dense.shape[0] + index_offset = numpy.arange(num_labels) * num_classes + labels_one_hot = numpy.zeros((num_labels, num_classes)) + labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 + return labels_one_hot + + +@deprecated(None, 'Please use tf.data to implement this functionality.') +def _extract_labels(f, one_hot=False, num_classes=10): + """Extract the labels into a 1D uint8 numpy array [index]. + + Args: + f: A file object that can be passed into a gzip reader. + one_hot: Does one hot encoding for the result. + num_classes: Number of classes for the one hot encoding. + + Returns: + labels: a 1D uint8 numpy array. + + Raises: + ValueError: If the bystream doesn't start with 2049. + """ + print('Extracting', f.name) + with gzip.GzipFile(fileobj=f) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError('Invalid magic number %d in MNIST label file: %s' % + (magic, f.name)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + if one_hot: + return _dense_to_one_hot(labels, num_classes) + return labels + + +class _DataSet(object): + """Container class for a _DataSet (deprecated). + + THIS CLASS IS DEPRECATED. + """ + + @deprecated(None, 'Please use alternatives such as official/mnist/_DataSet.py' + ' from tensorflow/models.') + def __init__(self, + images, + labels, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + seed=None): + """Construct a _DataSet. + + one_hot arg is used only if fake_data is true. `dtype` can be either + `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into + `[0, 1]`. Seed arg provides for convenient deterministic testing. + + Args: + images: The images + labels: The labels + fake_data: Ignore inages and labels, use fake data. + one_hot: Bool, return the labels as one hot vectors (if True) or ints (if + False). + dtype: Output image dtype. One of [uint8, float32]. `uint8` output has + range [0,255]. float32 output has range [0,1]. + reshape: Bool. If True returned images are returned flattened to vectors. + seed: The random seed to use. + """ + seed1, seed2 = random_seed.get_seed(seed) + # If op level seed is not set, use whatever graph level seed is returned + numpy.random.seed(seed1 if seed is None else seed2) + dtype = dtypes.as_dtype(dtype).base_dtype + if dtype not in (dtypes.uint8, dtypes.float32): + raise TypeError('Invalid image dtype %r, expected uint8 or float32' % + dtype) + if fake_data: + self._num_examples = 10000 + self.one_hot = one_hot + else: + assert images.shape[0] == labels.shape[0], ( + 'images.shape: %s labels.shape: %s' % (images.shape, labels.shape)) + self._num_examples = images.shape[0] + + # Convert shape from [num examples, rows, columns, depth] + # to [num examples, rows*columns] (assuming depth == 1) + if reshape: + assert images.shape[3] == 1 + images = images.reshape(images.shape[0], + images.shape[1] * images.shape[2]) + if dtype == dtypes.float32: + # Convert from [0, 255] -> [0.0, 1.0]. + images = images.astype(numpy.float32) + images = numpy.multiply(images, 1.0 / 255.0) + self._images = images + self._labels = labels + self._epochs_completed = 0 + self._index_in_epoch = 0 + + @property + def images(self): + return self._images + + @property + def labels(self): + return self._labels + + @property + def num_examples(self): + return self._num_examples + + @property + def epochs_completed(self): + return self._epochs_completed + + def next_batch(self, batch_size, fake_data=False, shuffle=True): + """Return the next `batch_size` examples from this data set.""" + if fake_data: + fake_image = [1] * 784 + if self.one_hot: + fake_label = [1] + [0] * 9 + else: + fake_label = 0 + return [fake_image for _ in xrange(batch_size) + ], [fake_label for _ in xrange(batch_size)] + start = self._index_in_epoch + # Shuffle for the first epoch + if self._epochs_completed == 0 and start == 0 and shuffle: + perm0 = numpy.arange(self._num_examples) + numpy.random.shuffle(perm0) + self._images = self.images[perm0] + self._labels = self.labels[perm0] + # Go to the next epoch + if start + batch_size > self._num_examples: + # Finished epoch + self._epochs_completed += 1 + # Get the rest examples in this epoch + rest_num_examples = self._num_examples - start + images_rest_part = self._images[start:self._num_examples] + labels_rest_part = self._labels[start:self._num_examples] + # Shuffle the data + if shuffle: + perm = numpy.arange(self._num_examples) + numpy.random.shuffle(perm) + self._images = self.images[perm] + self._labels = self.labels[perm] + # Start next epoch + start = 0 + self._index_in_epoch = batch_size - rest_num_examples + end = self._index_in_epoch + images_new_part = self._images[start:end] + labels_new_part = self._labels[start:end] + return numpy.concatenate((images_rest_part, images_new_part), + axis=0), numpy.concatenate( + (labels_rest_part, labels_new_part), axis=0) + else: + self._index_in_epoch += batch_size + end = self._index_in_epoch + return self._images[start:end], self._labels[start:end] + + +@deprecated(None, 'Please write your own downloading logic.') +def _maybe_download(filename, work_directory, source_url): + """Download the data from source url, unless it's already here. + + Args: + filename: string, name of the file in the directory. + work_directory: string, path to working directory. + source_url: url to download from if file doesn't exist. + + Returns: + Path to resulting file. + """ + if not gfile.Exists(work_directory): + gfile.MakeDirs(work_directory) + filepath = os.path.join(work_directory, filename) + if not gfile.Exists(filepath): + urllib.request.urlretrieve(source_url, filepath) + with gfile.GFile(filepath) as f: + size = f.size() + print('Successfully downloaded', filename, size, 'bytes.') + return filepath + + +@deprecated(None, 'Please use alternatives such as:' + ' tensorflow_datasets.load(\'mnist\')') +def read_data_sets(train_dir, + fake_data=False, + one_hot=False, + dtype=dtypes.float32, + reshape=True, + validation_size=5000, + seed=None, + source_url=DEFAULT_SOURCE_URL): + if fake_data: + + def fake(): + return _DataSet([], [], + fake_data=True, + one_hot=one_hot, + dtype=dtype, + seed=seed) + + train = fake() + validation = fake() + test = fake() + return _Datasets(train=train, validation=validation, test=test) + + if not source_url: # empty string check + source_url = DEFAULT_SOURCE_URL + + train_images_file = 'train-images-idx3-ubyte.gz' + train_labels_file = 'train-labels-idx1-ubyte.gz' + test_images_file = 't10k-images-idx3-ubyte.gz' + test_labels_file = 't10k-labels-idx1-ubyte.gz' + + local_file = _maybe_download(train_images_file, train_dir, + source_url + train_images_file) + with gfile.Open(local_file, 'rb') as f: + train_images = _extract_images(f) + + local_file = _maybe_download(train_labels_file, train_dir, + source_url + train_labels_file) + with gfile.Open(local_file, 'rb') as f: + train_labels = _extract_labels(f, one_hot=one_hot) + + local_file = _maybe_download(test_images_file, train_dir, + source_url + test_images_file) + with gfile.Open(local_file, 'rb') as f: + test_images = _extract_images(f) + + local_file = _maybe_download(test_labels_file, train_dir, + source_url + test_labels_file) + with gfile.Open(local_file, 'rb') as f: + test_labels = _extract_labels(f, one_hot=one_hot) + + if not 0 <= validation_size <= len(train_images): + raise ValueError( + 'Validation size should be between 0 and {}. Received: {}.'.format( + len(train_images), validation_size)) + + validation_images = train_images[:validation_size] + validation_labels = train_labels[:validation_size] + train_images = train_images[validation_size:] + train_labels = train_labels[validation_size:] + + options = dict(dtype=dtype, reshape=reshape, seed=seed) + + train = _DataSet(train_images, train_labels, **options) + validation = _DataSet(validation_images, validation_labels, **options) + test = _DataSet(test_images, test_labels, **options) + + return _Datasets(train=train, validation=validation, test=test)