From 9f829f297922742cf778201c777ec56b31bd73aa Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 20 Jun 2013 09:26:05 -0400 Subject: [PATCH 001/417] pep8 --- code/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 6a3390bf..586d7d32 100644 --- a/code/test.py +++ b/code/test.py @@ -24,7 +24,8 @@ def test_logistic_cg(): logistic_cg.cg_optimization_mnist(n_epochs=10) except ImportError: from nose.plugins.skip import SkipTest - raise SkipTest('SciPy not available. Needed for the logistic_cg example.') + raise SkipTest( + 'SciPy not available. Needed for the logistic_cg example.') def test_mlp(): @@ -34,6 +35,7 @@ def test_mlp(): def test_convolutional_mlp(): convolutional_mlp.evaluate_lenet5(n_epochs=1, nkerns=[5, 5]) + def test_dA(): dA.test_dA(training_epochs=1, output_folder='tmp_dA_plots') @@ -50,9 +52,11 @@ def test_rbm(): rbm.test_rbm(training_epochs=1, batch_size=300, n_chains=1, n_samples=1, n_hidden=20, output_folder='tmp_rbm_plots') + def test_rnnrbm(): rnnrbm.test_rnnrbm(num_epochs=1) + def speed(): """ This fonction modify the configuration theano and don't restore it! From 061d8358ba8d40d2bc70a1407a663888c4cd0381 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 20 Jun 2013 09:26:16 -0400 Subject: [PATCH 002/417] update timming to faster case in float32. --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 586d7d32..63a4f514 100644 --- a/code/test.py +++ b/code/test.py @@ -78,7 +78,7 @@ def speed(): expected_times_64 = numpy.asarray([10.3, 23.7, 78.1, 73.7, 116.4, 346.9, 381.9, 558.1, 186.3]) - expected_times_32 = numpy.asarray([11.6, 29.6, 47.2, 66.5, 71, + expected_times_32 = numpy.asarray([11.6, 29.6, 44.2, 66.5, 71, 191.2, 226.8, 432.8, 176.2]) # Number with just 1 decimal are new value that are faster with From 509b7c45b6c7e4640a6d8d121684af60bd15a9cf Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 20 Jun 2013 09:27:10 -0400 Subject: [PATCH 003/417] Add name to fct. --- code/SdA.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index 20c20426..71d95463 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -263,21 +263,24 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): self.x: train_set_x[index * batch_size: (index + 1) * batch_size], self.y: train_set_y[index * batch_size: - (index + 1) * batch_size]}) + (index + 1) * batch_size]}, + name='train') test_score_i = theano.function([index], self.errors, givens={ self.x: test_set_x[index * batch_size: (index + 1) * batch_size], self.y: test_set_y[index * batch_size: - (index + 1) * batch_size]}) + (index + 1) * batch_size]}, + name='test') valid_score_i = theano.function([index], self.errors, givens={ self.x: valid_set_x[index * batch_size: (index + 1) * batch_size], self.y: valid_set_y[index * batch_size: - (index + 1) * batch_size]}) + (index + 1) * batch_size]}, + name='valid') # Create a function that scans the entire validation set def valid_score(): From c5ce8ff2f0a57091a1f9c11a439ffa68f3e5ca4e Mon Sep 17 00:00:00 2001 From: Paul Hobbs Date: Sun, 18 Aug 2013 19:56:17 -0700 Subject: [PATCH 004/417] Fix typo in mlp.txt The example code to compute the cost of the linear combination of L1 and L2 regularization penalties didn't match the code below. They referred to variables `L1` and `L2`, which should actually be `classifier.L1` and `classifier.L2`. --- doc/mlp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index 20794335..4772d1ee 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -289,8 +289,8 @@ The code that computes the new cost is: # the model plus the regularization terms (L1 and L2); cost is expressed # here symbolically cost = classifier.negative_log_likelihood(y) \ - + L1_reg * L1 \ - + L2_reg * L2_sqr + + L1_reg * classifier.L1 \ + + L2_reg * classifier.L2_sqr We then update the parameters of the model using the gradient. This code is From d0bc7d9010ac5fee66f4ef8e1841d8939fce913d Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Fri, 23 Aug 2013 14:11:41 -0400 Subject: [PATCH 005/417] Remove test for mcrbm that had no tutorial and depended on pylearn Moves the hmc code from mcrbm/ to hmc/ --- .travis.yml | 19 +- code/{mcrbm => hmc}/__init__.py | 0 code/{mcrbm => hmc}/hmc.py | 0 code/{mcrbm => hmc}/test_hmc.py | 0 code/mcrbm/mcrbm.py | 781 -------------------------------- code/mcrbm/test_mcrbm.py | 147 ------ doc/hmc.txt | 6 +- 7 files changed, 12 insertions(+), 941 deletions(-) rename code/{mcrbm => hmc}/__init__.py (100%) rename code/{mcrbm => hmc}/hmc.py (100%) rename code/{mcrbm => hmc}/test_hmc.py (100%) delete mode 100644 code/mcrbm/mcrbm.py delete mode 100644 code/mcrbm/test_mcrbm.py diff --git a/.travis.yml b/.travis.yml index 7ab40a13..2a2f88a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,16 +31,15 @@ env: - PART="test.py:test_rbm" - PART="-e test.py" -#569.882s #10 code.test.test_rbm OK -#298.992s #9 code.test.test_dbn OK -#268.901s #8 code.test.test_SdA OK -#67.292s #7 code.test.test_dA OK -#27.485s #5 code.test.test_mlp OK -#26.204s #6 code.test.test_convolutional_mlp OK -#14.676s #4 code.test.test_logistic_cg OK -#10.66s #3 code.test.test_logistic_sgd OK -#5.795s #1 code.mcrbm.test_hmc.test_hmc OK -#0.0s #2 code.mcrbm.test_mcrbm.test_reproduce_ranzato_hinton_2010 FAILED TEST +#569.882s #9 code.test.test_rbm OK +#298.992s #8 code.test.test_dbn OK +#268.901s #7 code.test.test_SdA OK +#67.292s #6 code.test.test_dA OK +#27.485s #4 code.test.test_mlp OK +#26.204s #5 code.test.test_convolutional_mlp OK +#14.676s #3 code.test.test_logistic_cg OK +#10.66s #2 code.test.test_logistic_sgd OK +#5.795s #1 code.hmc.test_hmc.test_hmc OK script: - cd data diff --git a/code/mcrbm/__init__.py b/code/hmc/__init__.py similarity index 100% rename from code/mcrbm/__init__.py rename to code/hmc/__init__.py diff --git a/code/mcrbm/hmc.py b/code/hmc/hmc.py similarity index 100% rename from code/mcrbm/hmc.py rename to code/hmc/hmc.py diff --git a/code/mcrbm/test_hmc.py b/code/hmc/test_hmc.py similarity index 100% rename from code/mcrbm/test_hmc.py rename to code/hmc/test_hmc.py diff --git a/code/mcrbm/mcrbm.py b/code/mcrbm/mcrbm.py deleted file mode 100644 index a5ea8511..00000000 --- a/code/mcrbm/mcrbm.py +++ /dev/null @@ -1,781 +0,0 @@ -""" -This file implements the Mean & Covariance RBM discussed in - - Ranzato, M. and Hinton, G. E. (2010) - Modeling pixel means and covariances using factored third-order Boltzmann machines. - IEEE Conference on Computer Vision and Pattern Recognition. - -and performs one of the experiments on CIFAR-10 discussed in that paper. There are some minor -discrepancies between the paper and the accompanying code (train_mcRBM.py), and the -accompanying code has been taken to be correct in those cases because I couldn't get things to -work otherwise. - - -Math -==== - -Energy of "covariance RBM" - - E = -0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i C_{if} v_i )^2 - = -0.5 \sum_f (\sum_k P_{fk} h_k) ( \sum_i C_{if} v_i )^2 - "vector element f" "vector element f" - -In some parts of the paper, the P matrix is chosen to be a diagonal matrix with non-positive -diagonal entries, so it is helpful to see this as a simpler equation: - - E = \sum_f h_f ( \sum_i C_{if} v_i )^2 - - - -Version in paper ----------------- - -Full Energy of the Mean and Covariance RBM, with -:math:`h_k = h_k^{(c)}`, -:math:`g_j = h_j^{(m)}`, -:math:`b_k = b_k^{(c)}`, -:math:`c_j = b_j^{(m)}`, -:math:`U_{if} = C_{if}`, - - E (v, h, g) = - - 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i (U_{if} v_i) / |U_{.f}|*|v| )^2 - - \sum_k b_k h_k - + 0.5 \sum_i v_i^2 - - \sum_j \sum_i W_{ij} g_j v_i - - \sum_j c_j g_j - -For the energy function to correspond to a probability distribution, P must be non-positive. P -is initialized to be a diagonal or a topological pooling matrix, and in our experience it can -be left as such because even in the paper it has a very low learning rate, and is only allowed -to be updated after the filters in U are learned (in effect). - -Version in published train_mcRBM code -------------------------------------- - -The train_mcRBM file implements learning in a similar but technically different Energy function: - - E (v, h, g) = - 0.5 \sum_f \sum_k P_{fk} h_k (\sum_i U_{if} v_i / sqrt(\sum_i v_i^2/I + 0.5))^2 - - \sum_k b_k h_k - + 0.5 \sum_i v_i^2 - - \sum_j \sum_i W_{ij} g_j v_i - - \sum_j c_j g_j - -There are two differences with respect to the paper: - - - 'v' is not normalized by its length, but rather it is normalized to have length close to - the square root of the number of its components. The variable called 'small' that - "avoids division by zero" is orders larger than machine precision, and is on the order of - the normalized sum-of-squares, so I've included it in the Energy function. - - - 'U' is also not normalized by its length. U is initialized to have columns that are - shorter than unit-length (approximately 0.2 with the 105 principle components in the - train_mcRBM data). During training, the columns of U are constrained manually to have - equal lengths (see the use of normVF), but Euclidean norm is allowed to change. During - learning it quickly converges towards 1 and then exceeds 1. It does not seem like this - column-wise normalization of U is justified by maximum-likelihood, I have no intuition - for why it is used. - - -Version in this code --------------------- - -This file implements the same algorithm as the train_mcRBM code, except that the P matrix is -omitted for clarity, and replaced analytically with a negative identity matrix. - - E (v, h, g) = - + 0.5 \sum_k h_k (\sum_i U_{ik} v_i / sqrt(\sum_i v_i^2/I + 0.5))^2 - - \sum_k b_k h_k - + 0.5 \sum_i v_i^2 - - \sum_j \sum_i W_{ij} g_j v_i - - \sum_j c_j g_j - - E (v, h, g) = - - 0.5 \sum_f \sum_k P_{fk} h_k (\sum_i U_{if} v_i / sqrt(\sum_i v_i^2/I + 0.5))^2 - - \sum_k b_k h_k - + 0.5 \sum_i v_i^2 - - \sum_j \sum_i W_{ij} g_j v_i - - \sum_j c_j g_j - - - -Conventions in this file -======================== - -This file contains some global functions, as well as a class (MeanCovRBM) that makes using them a little -more convenient. - - -Global functions like `free_energy` work on an mcRBM as parametrized in a particular way. -Suppose we have - - I input dimensions, - - F squared filters, - - J mean variables, and - - K covariance variables. - -The mcRBM is parametrized by 6 variables: - - - `P`, a matrix whose rows indicate covariance filter groups (F x K) - - `U`, a matrix whose rows are visible covariance directions (I x F) - - `W`, a matrix whose rows are visible mean directions (I x J) - - `b`, a vector of hidden covariance biases (K) - - `c`, a vector of hidden mean biases (J) - -Matrices are generally layed out and accessed according to a C-order convention. - -""" - -# -# WORKING NOTES -# THIS DERIVATION IS BASED ON THE ** PAPER ** ENERGY FUNCTION -# NOT THE ENERGY FUNCTION IN THE CODE!!! -# -# Free energy is the marginal energy of visible units -# Recall: -# Q(x) = exp(-E(x))/Z ==> -log(Q(x)) - log(Z) = E(x) -# -# -# E (v, h, g) = -# - 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i U_{if} v_i )^2 / |U_{*f}|^2 |v|^2 -# - \sum_k b_k h_k -# + 0.5 \sum_i v_i^2 -# - \sum_j \sum_i W_{ij} g_j v_i -# - \sum_j c_j g_j -# - \sum_i a_i v_i -# -# -# Derivation, in which partition functions are ignored. -# -# E(v) = -\log(Q(v)) -# = -\log( \sum_{h,g} Q(v,h,g)) -# = -\log( \sum_{h,g} exp(-E(v,h,g))) -# = -\log( \sum_{h,g} exp(- -# - 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i U_{if} v_i )^2 / (|U_{*f}| * |v|) -# - \sum_k b_k h_k -# + 0.5 \sum_i v_i^2 -# - \sum_j \sum_i W_{ij} g_j v_i -# - \sum_j c_j g_j -# - \sum_i a_i v_i )) -# -# Get rid of double negs in exp -# = -\log( \sum_{h} exp( -# + 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i U_{if} v_i )^2 / (|U_{*f}| * |v|) -# + \sum_k b_k h_k -# - 0.5 \sum_i v_i^2 -# ) * \sum_{g} exp( -# + \sum_j \sum_i W_{ij} g_j v_i -# + \sum_j c_j g_j)) -# - \sum_i a_i v_i -# -# Break up log -# = -\log( \sum_{h} exp( -# + 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i U_{if} v_i )^2 / (|U_{*f}|*|v|) -# + \sum_k b_k h_k -# )) -# -\log( \sum_{g} exp( -# + \sum_j \sum_i W_{ij} g_j v_i -# + \sum_j c_j g_j ))) -# + 0.5 \sum_i v_i^2 -# - \sum_i a_i v_i -# -# Use domain h is binary to turn log(sum(exp(sum...))) into sum(log(.. -# = -\log(\sum_{h} exp( -# + 0.5 \sum_f \sum_k P_{fk} h_k ( \sum_i U_{if} v_i )^2 / (|U_{*f}|* |v|) -# + \sum_k b_k h_k -# )) -# - \sum_{j} \log(1 + exp(\sum_i W_{ij} v_i + c_j )) -# + 0.5 \sum_i v_i^2 -# - \sum_i a_i v_i -# -# = - \sum_{k} \log(1 + exp(b_k + 0.5 \sum_f P_{fk}( \sum_i U_{if} v_i )^2 / (|U_{*f}|*|v|))) -# - \sum_{j} \log(1 + exp(\sum_i W_{ij} v_i + c_j )) -# + 0.5 \sum_i v_i^2 -# - \sum_i a_i v_i -# -# For negative-one-diagonal P this gives: -# -# = - \sum_{k} \log(1 + exp(b_k - 0.5 \sum_i (U_{ik} v_i )^2 / (|U_{*k}|*|v|))) -# - \sum_{j} \log(1 + exp(\sum_i W_{ij} v_i + c_j )) -# + 0.5 \sum_i v_i^2 -# - \sum_i a_i v_i - -import sys, os, logging -import numpy as np -import numpy - -import theano -from theano import function, shared, dot -from theano import tensor as TT -floatX = theano.config.floatX - -sharedX = lambda X, name : shared(numpy.asarray(X, dtype=floatX), name=name) - -import pylearn -from pylearn.sampling.hmc import HMC_sampler -from pylearn.io import image_tiling -from pylearn.gd.sgd import sgd_updates -import pylearn.dataset_ops.image_patches - -########################################### -# -# Candidates for factoring -# -########################################### - -def l1(X): - """ - :param X: TensorType variable - - :rtype: TensorType scalar - - :returns: the sum of absolute values of the terms in X - - :math: \sum_i |X_i| - - Where i is an appropriately dimensioned index. - - """ - return abs(X).sum() - -def l2(X): - """ - :param X: TensorType variable - - :rtype: TensorType scalar - - :returns: the sum of absolute values of the terms in X - - :math: \sqrt{ \sum_i X_i^2 } - - Where i is an appropriately dimensioned index. - - """ - return TT.sqrt((X**2).sum()) - -def contrastive_cost(free_energy_fn, pos_v, neg_v): - """ - :param free_energy_fn: lambda (TensorType matrix MxN) -> TensorType vector of M free energies - :param pos_v: TensorType matrix MxN of M "positive phase" particles - :param neg_v: TensorType matrix MxN of M "negative phase" particles - - :returns: TensorType scalar that's the sum of the difference of free energies - - :math: \sum_i free_energy(pos_v[i]) - free_energy(neg_v[i]) - - """ - return (free_energy_fn(pos_v) - free_energy_fn(neg_v)).sum() - -def contrastive_grad(free_energy_fn, pos_v, neg_v, wrt, other_cost=0): - """ - :param free_energy_fn: lambda (TensorType matrix MxN) -> TensorType vector of M free energies - :param pos_v: positive-phase sample of visible units - :param neg_v: negative-phase sample of visible units - :param wrt: TensorType variables with respect to which we want gradients (similar to the - 'wrt' argument to tensor.grad) - :param other_cost: TensorType scalar - - :returns: TensorType variables for the gradient on each of the 'wrt' arguments - - - :math: Cost = other_cost + \sum_i free_energy(pos_v[i]) - free_energy(neg_v[i]) - :math: d Cost / dW for W in `wrt` - - - This function is similar to tensor.grad - it returns the gradient[s] on a cost with respect - to one or more parameters. The difference between tensor.grad and this function is that - the negative phase term (`neg_v`) is considered constant, i.e. d `Cost` / d `neg_v` = 0. - This is desirable because `neg_v` might be the result of a sampling expression involving - some of the parameters, but the contrastive divergence algorithm does not call for - backpropagating through the sampling procedure. - - Warning - if other_cost depends on pos_v or neg_v and you *do* want to backpropagate from - the `other_cost` through those terms, then this function is inappropriate. In that case, - you should call tensor.grad separately for the other_cost and add the gradient expressions - you get from ``contrastive_grad(..., other_cost=0)`` - - """ - cost=contrastive_cost(free_energy_fn, pos_v, neg_v) - if other_cost: - cost = cost + other_cost - return theano.tensor.grad(cost, - wrt=wrt, - consider_constant=[neg_v]) - -########################################### -# -# Expressions that are mcRBM-specific -# -########################################### - -class mcRBM(object): - """Light-weight class that provides the math related to inference - - Attributes: - - - U - the covariance filters (theano shared variable) - - W - the mean filters (theano shared variable) - - a - the visible bias (theano shared variable) - - b - the covariance bias (theano shared variable) - - c - the mean bias (theano shared variable) - - """ - def __init__(self, U, W, a, b, c): - self.U = U - self.W = W - self.a = a - self.b = b - self.c = c - - def hidden_cov_units_preactivation_given_v(self, v, small=0.5): - """Return argument to the sigmoid that would give mean of covariance hid units - return b - 0.5 * dot(v/||v||, U)**2 - """ - unit_v = v / (TT.sqrt(TT.mean(v**2, axis=1)+small)).dimshuffle(0,'x') # adjust row norm - return self.b - 0.5 * dot(unit_v, self.U)**2 - - def free_energy_terms_given_v(self, v): - """Returns theano expression for the terms that are added to form the free energy of - visible vector `v` in an mcRBM. - - 1. Free energy related to covariance hiddens - 2. Free energy related to mean hiddens - 3. Free energy related to L2-Norm of `v` - 4. Free energy related to projection of `v` onto biases `a` - """ - t0 = -TT.sum(TT.nnet.softplus(self.hidden_cov_units_preactivation_given_v(v)),axis=1) - t1 = -TT.sum(TT.nnet.softplus(self.c + dot(v,self.W)), axis=1) - t2 = 0.5 * TT.sum(v**2, axis=1) - t3 = -TT.dot(v, self.a) - return [t0, t1, t2, t3] - - def free_energy_given_v(self, v): - """Returns theano expression for free energy of visible vector `v` in an mcRBM - """ - return TT.add(*self.free_energy_terms_given_v(v)) - - def expected_h_g_given_v(self, v): - """Returns tuple (`h`, `g`) of theano expression conditional expectations in an mcRBM. - - `h` is the conditional on the covariance units. - `g` is the conditional on the mean units. - - """ - h = TT.nnet.sigmoid(self.hidden_cov_units_preactivation_given_v(v)) - g = TT.nnet.sigmoid(self.c + dot(v,self.W)) - return (h, g) - - def n_visible_units(self): - """Return the number of visible units of this RBM - - For an RBM made from shared variables, this will return an integer, - for a purely symbolic RBM this will return a theano expression. - - """ - try: - return self.W.get_value(borrow=True).shape[0] - except AttributeError: - return self.W.shape[0] - - def n_hidden_cov_units(self): - """Return the number of hidden units for the covariance in this RBM - - For an RBM made from shared variables, this will return an integer, - for a purely symbolic RBM this will return a theano expression. - - """ - try: - return self.U.get_value(borrow=True).shape[1] - except AttributeError: - return self.U.shape[1] - - def n_hidden_mean_units(self): - """Return the number of hidden units for the mean in this RBM - - For an RBM made from shared variables, this will return an integer, - for a purely symbolic RBM this will return a theano expression. - - """ - try: - return self.W.get_value(borrow=True).shape[1] - except AttributeError: - return self.W.shape[1] - - def CD1_sampler(self, v, n_particles, n_visible=None, rng=8923984): - """Return a symbolic negative-phase particle obtained by simulating the Hamiltonian - associated with the energy function. - """ - #TODO: why not expose all HMC arguments somehow? - if not hasattr(rng, 'randn'): - rng = np.random.RandomState(rng) - if n_visible is None: - n_visible = self.n_visible_units() - - # create a dummy hmc object because we want to use *some* of it - hmc = HMC_sampler.new_from_shared_positions( - shared_positions=v, # v is not shared, so some functionality will not work - energy_fn=self.free_energy_given_v, - seed=int(rng.randint(2**30)), - shared_positions_shape=(n_particles,n_visible), - compile_simulate=False) - updates = dict(hmc.updates()) - final_p = updates.pop(v) - return hmc, final_p, updates - - def sampler(self, n_particles, n_visible=None, rng=7823748): - """Return an `HMC_sampler` that will draw samples from the distribution over visible - units specified by this RBM. - - :param n_particles: this many parallel chains will be simulated. - :param rng: seed or numpy RandomState object to initialize particles, and to drive the simulation. - """ - #TODO: why not expose all HMC arguments somehow? - #TODO: Consider returning a sample kwargs for passing to HMC_sampler? - if not hasattr(rng, 'randn'): - rng = np.random.RandomState(rng) - if n_visible is None: - n_visible = self.n_visible_units() - rval = HMC_sampler.new_from_shared_positions( - shared_positions = sharedX( - rng.randn( - n_particles, - n_visible), - name='particles'), - energy_fn=self.free_energy_given_v, - seed=int(rng.randint(2**30))) - return rval - - def params(self): - """Return the elements of [U,W,a,b,c] that are shared variables - - WRITEME : a *prescriptive* definition of this method suitable for mention in the API - doc. - - """ - return list(self._params) - - @classmethod - def alloc(cls, n_I, n_K, n_J, rng = 8923402190, - U_range=0.02, - W_range=0.05, - a_ival=0, - b_ival=2, - c_ival=-2): - """ - Return a MeanCovRBM instance with randomly-initialized shared variable parameters. - - :param n_I: input dimensionality - :param n_K: number of covariance hidden units - :param n_J: number of mean filters (linear) - :param rng: seed or numpy RandomState object to initialize parameters - - :note: - Constants for initial ranges and values taken from train_mcRBM.py. - """ - if not hasattr(rng, 'randn'): - rng = np.random.RandomState(rng) - - rval = cls( - U = sharedX(U_range * rng.randn(n_I, n_K),'U'), - W = sharedX(W_range * rng.randn(n_I, n_J),'W'), - a = sharedX(np.ones(n_I)*a_ival,'a'), - b = sharedX(np.ones(n_K)*b_ival,'b'), - c = sharedX(np.ones(n_J)*c_ival,'c'),) - rval._params = [rval.U, rval.W, rval.a, rval.b, rval.c] - return rval - -def topological_connectivity(out_shape=(12,12), window_shape=(3,3), window_stride=(2,2), - **kwargs): - - in_shape = (window_stride[0] * out_shape[0], - window_stride[1] * out_shape[1]) - - rval = numpy.zeros(in_shape + out_shape, dtype=theano.config.floatX) - A,B,C,D = rval.shape - - # for each output position (out_r, out_c) - for out_r in range(out_shape[0]): - for out_c in range(out_shape[1]): - # for each window position (win_r, win_c) - for win_r in range(window_shape[0]): - for win_c in range(window_shape[1]): - # add 1 to the corresponding input location - in_r = out_r * window_stride[0] + win_r - in_c = out_c * window_stride[1] + win_c - rval[in_r%A, in_c%B, out_r%C, out_c%D] += 1 - - # This normalization algorithm is a guess, based on inspection of the matrix loaded from - # see CVPR2010paper_material/topo2D_3x3_stride2_576filt.mat - rval = rval.reshape((A*B, C*D)) - rval = (rval.T / rval.sum(axis=1)).T - - rval /= rval.sum(axis=0) - return rval - -class mcRBM_withP(mcRBM): - """Light-weight class that provides the math related to inference - - Attributes: - - - U - the covariance filters (theano shared variable) - - W - the mean filters (theano shared variable) - - a - the visible bias (theano shared variable) - - b - the covariance bias (theano shared variable) - - c - the mean bias (theano shared variable) - - """ - def __init__(self, U, W, a, b, c, P): - self.P = P - super(mcRBM_withP, self).__init__(U,W,a,b,c) - - def hidden_cov_units_preactivation_given_v(self, v, small=0.5): - """Return argument to the sigmoid that would give mean of covariance hid units - - See the math at the top of this file for what 'adjusted' means. - - return b - 0.5 * dot(adjusted(v), U)**2 - """ - unit_v = v / (TT.sqrt(TT.mean(v**2, axis=1)+small)).dimshuffle(0,'x') # adjust row norm - return self.b + 0.5 * dot(dot(unit_v, self.U)**2, self.P) - - def n_hidden_cov_units(self): - """Return the number of hidden units for the covariance in this RBM - - For an RBM made from shared variables, this will return an integer, - for a purely symbolic RBM this will return a theano expression. - - """ - try: - return self.P.get_value(borrow=True).shape[1] - except AttributeError: - return self.P.shape[1] - - @classmethod - def alloc(cls, n_I, n_K, n_J, *args, **kwargs): - """ - Return a MeanCovRBM instance with randomly-initialized shared variable parameters. - - :param n_I: input dimensionality - :param n_K: number of covariance hidden units - :param n_J: number of mean filters (linear) - :param rng: seed or numpy RandomState object to initialize parameters - - :note: - Constants for initial ranges and values taken from train_mcRBM.py. - """ - return cls.alloc_with_P( - -numpy.eye((n_K, n_K)).astype(theano.config.floatX), - n_I, - n_J, - *args, **kwargs) - - @classmethod - def alloc_topo_P(cls, n_I, n_J, p_out_shape=(12,12), p_win_shape=(3,3), p_win_stride=(2,2), - **kwargs): - return cls.alloc_with_P( - -topological_connectivity(p_out_shape, p_win_shape, p_win_stride), - n_I=n_I, n_J=n_J, **kwargs) - - @classmethod - def alloc_with_P(cls, Pval, n_I, n_J, rng = 8923402190, - U_range=0.02, - W_range=0.05, - a_ival=0, - b_ival=2, - c_ival=-2): - n_F, n_K = Pval.shape - if not hasattr(rng, 'randn'): - rng = np.random.RandomState(rng) - rval = cls( - U = sharedX(U_range * rng.randn(n_I, n_F),'U'), - W = sharedX(W_range * rng.randn(n_I, n_J),'W'), - a = sharedX(np.ones(n_I)*a_ival,'a'), - b = sharedX(np.ones(n_K)*b_ival,'b'), - c = sharedX(np.ones(n_J)*c_ival,'c'), - P = sharedX(Pval, 'P'),) - rval._params = [rval.U, rval.W, rval.a, rval.b, rval.c, rval.P] - return rval - -class mcRBMTrainer(object): - """Light-weight class encapsulating math for mcRBM training - - Attributes: - - rbm - an mcRBM instance - - sampler - an HMC_sampler instance - - normVF - geometrically updated norm of U matrix columns (shared var) - - learn_rate - SGD learning rate [un-annealed] - - learn_rate_multipliers - the learning rates for each of the parameters of the rbm (in - order corresponding to what's returned by ``rbm.params()``) - - l1_penalty - float or TensorType scalar to modulate l1 penalty of rbm.U and rbm.W - - iter - number of cd_updates (shared var) - used to anneal the effective learn_rate - - lr_anneal_start - scalar or TensorType scalar - iter at which time to start decreasing - the learning rate proportional to 1/iter - - """ - # TODO: accept a GD algo as an argument? - @classmethod - def alloc_for_P(cls, rbm, visible_batch, batchsize, initial_lr_per_example=0.075, rng=234, - l1_penalty=0, - l1_penalty_start=0, - learn_rate_multipliers=None, - lr_anneal_start=2000, - p_training_start=4000, - p_training_lr=0.02, - persistent_chains=True - ): - if learn_rate_multipliers is None: - p_lr = sharedX(0.0, 'P_lr_multiplier') - learn_rate_multipliers = [2, .2, .02, .1, .02, p_lr] - else: - p_lr = None - rval = cls.alloc(rbm, visible_batch, batchsize, initial_lr_per_example, rng, l1_penalty, - l1_penalty_start, learn_rate_multipliers, lr_anneal_start, persistent_chains) - - rval.p_mask = sharedX((rbm.P.get_value(borrow=True) != 0).astype('float32'), 'p_mask') - - rval.p_lr = p_lr - rval.p_training_start=p_training_start - rval.p_training_lr=p_training_lr - return rval - - - @classmethod - def alloc(cls, rbm, visible_batch, batchsize, initial_lr_per_example=0.075, rng=234, - l1_penalty=0, - l1_penalty_start=0, - learn_rate_multipliers=[2, .2, .02, .1, .02], - lr_anneal_start=2000, - persistent_chains=True - ): - - """ - :param rbm: mcRBM instance to train - :param visible_batch: TensorType variable for training data - :param batchsize: the number of rows in visible_batch - :param initial_lr_per_example: the learning rate (may be annealed) - :param rng: seed or RandomState to initialze PCD sampler - :param l1_penalty: see class doc - :param learn_rate_multipliers: see class doc - :param lr_anneal_start: see class doc - """ - #TODO: :param lr_anneal_iter: the iteration at which 1/t annealing will begin - - #TODO: get batchsize from visible_batch?? - # allocates shared var for negative phase particles - - - # TODO: should normVF be initialized to match the size of rbm.U ? - - if (l1_penalty_start > 0) and (l1_penalty != 0.0): - effective_l1_penalty = sharedX(0.0, 'effective_l1_penalty') - else: - effective_l1_penalty = l1_penalty - - if persistent_chains: - sampler = rbm.sampler(batchsize, rng=rng) - else: - sampler = None - - return cls( - rbm=rbm, - batchsize=batchsize, - visible_batch=visible_batch, - sampler=sampler, - normVF=sharedX(1.0, 'normVF'), - learn_rate=sharedX(initial_lr_per_example/batchsize, 'learn_rate'), - iter=sharedX(0, 'iter'), - effective_l1_penalty=effective_l1_penalty, - l1_penalty=l1_penalty, - l1_penalty_start=l1_penalty_start, - learn_rate_multipliers=learn_rate_multipliers, - lr_anneal_start=lr_anneal_start, - persistent_chains=persistent_chains,) - - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - def normalize_U(self, new_U): - """ - :param new_U: a proposed new value for rbm.U - - :returns: a pair of TensorType variables: - a corrected new value for U, and a new value for self.normVF - - This is a weird normalization procedure, but the sample code for the paper has it, and - it seems to be important. - """ - U_norms = TT.sqrt((new_U**2).sum(axis=0)) - new_normVF = .95 * self.normVF + .05 * TT.mean(U_norms) - return (new_U * new_normVF / U_norms), new_normVF - - def contrastive_grads(self, neg_v = None): - """Return the contrastive divergence gradients on the parameters of self.rbm """ - if neg_v is None: - neg_v = self.sampler.positions - return contrastive_grad( - free_energy_fn=self.rbm.free_energy_given_v, - pos_v=self.visible_batch, - neg_v=neg_v, - wrt = self.rbm.params(), - other_cost=(l1(self.rbm.U)+l1(self.rbm.W)) * self.effective_l1_penalty) - - def cd_updates(self): - """ - Return a dictionary of shared variable updates that implements contrastive divergence - learning by stochastic gradient descent with an annealed learning rate. - """ - - ups = {} - - if self.persistent_chains: - grads = self.contrastive_grads() - ups.update(dict(self.sampler.updates())) - else: - cd1_sampler, final_p, cd1_updates = self.rbm.CD1_sampler(self.visible_batch, - self.batchsize) - self._last_cd1_sampler = cd1_sampler # hacked in here for the unit test - #ignore the cd1_sampler - grads = self.contrastive_grads(neg_v = final_p) - ups.update(dict(cd1_updates)) - - - # contrastive divergence updates - # TODO: sgd_updates is a particular optization algo (others are possible) - # parametrize so that algo is plugin - # the normalization normVF might be sgd-specific though... - - # TODO: when sgd has an annealing schedule, this should - # go through that mechanism. - - lr = TT.clip( - self.learn_rate * TT.cast(self.lr_anneal_start / (self.iter+1), floatX), - 0.0, #min - self.learn_rate) #max - - ups.update(dict(sgd_updates( - self.rbm.params(), - grads, - stepsizes=[a*lr for a in self.learn_rate_multipliers]))) - - ups[self.iter] = self.iter + 1 - - - # add trainer updates (replace CD update of U) - ups[self.rbm.U], ups[self.normVF] = self.normalize_U(ups[self.rbm.U]) - - #l1_updates: - if (self.l1_penalty_start > 0) and (self.l1_penalty != 0.0): - ups[self.effective_l1_penalty] = TT.switch( - self.iter >= self.l1_penalty_start, - self.l1_penalty, - 0.0) - - if getattr(self,'p_lr', None): - ups[self.p_lr] = TT.switch(self.iter > self.p_training_start, - self.p_training_lr, - 0) - new_P = ups[self.rbm.P] * self.p_mask - no_pos_P = TT.switch(new_P<0, new_P, 0) - ups[self.rbm.P] = - no_pos_P / no_pos_P.sum(axis=0) #normalize to that columns sum 1 - - return ups - diff --git a/code/mcrbm/test_mcrbm.py b/code/mcrbm/test_mcrbm.py deleted file mode 100644 index a2a2ae69..00000000 --- a/code/mcrbm/test_mcrbm.py +++ /dev/null @@ -1,147 +0,0 @@ -import cPickle - -import numpy - -from pylearn.algorithms.mcRBM import mcRBM, mcRBMTrainer -from pylearn.dataset_ops import image_patches -import pylearn.datasets.cifar10 - -import theano -from theano import tensor - - -def l2(X): - return numpy.sqrt((X ** 2).sum()) - - -def _default_rbm_alloc(n_I, n_K=256, n_J=100): - return mcRBM.alloc(n_I, n_K, n_J) - - -def _default_trainer_alloc(rbm, train_batch, batchsize, initial_lr_per_example, - l1_penalty, l1_penalty_start, persistent_chains): - return mcRBMTrainer.alloc(rbm, train_batch, batchsize, - l1_penalty=l1_penalty, - l1_penalty_start=l1_penalty_start, - persistent_chains=persistent_chains) - - -def test_reproduce_ranzato_hinton_2010(dataset='MAR', - n_train_iters=5000, - rbm_alloc=_default_rbm_alloc, - trainer_alloc=_default_trainer_alloc, - lr_per_example=.075, - l1_penalty=1e-3, - l1_penalty_start=1000, - persistent_chains=True, - ): - - batchsize = 128 - ## specific to MAR dataset ## - n_vis = 105 - n_patches = 10240 - epoch_size = n_patches - - tile = image_patches.save_filters_of_ranzato_hinton_2010 - - batch_idx = tensor.iscalar() - batch_range = batch_idx * batchsize + numpy.arange(batchsize) - - train_batch = image_patches.ranzato_hinton_2010_op(batch_range) - - imgs_fn = theano.function([batch_idx], outputs=train_batch) - - trainer = trainer_alloc( - rbm_alloc(n_I=n_vis), - train_batch, - batchsize, - initial_lr_per_example=lr_per_example, - l1_penalty=l1_penalty, - l1_penalty_start=l1_penalty_start, - persistent_chains=persistent_chains) - rbm = trainer.rbm - - if persistent_chains: - grads = trainer.contrastive_grads() - learn_fn = theano.function([batch_idx], - outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)], - updates=trainer.cd_updates()) - else: - learn_fn = theano.function([batch_idx], outputs=[], - updates=trainer.cd_updates()) - - if persistent_chains: - smplr = trainer.sampler - else: - smplr = trainer._last_cd1_sampler - - if dataset == 'cifar10patches8x8': - cPickle.dump( - pylearn.dataset_ops.cifar10.random_cifar_patches_pca( - n_vis, None, 'float32', n_patches, R, C,), - open('test_mcRBM.pca.pkl', 'w')) - - print "Learning..." - last_epoch = -1 - for jj in xrange(n_train_iters): - epoch = jj * batchsize / epoch_size - - print_jj = epoch != last_epoch - last_epoch = epoch - - if print_jj: - tile(imgs_fn(jj), "imgs_%06i.png" % jj) - if persistent_chains: - tile(smplr.positions.get_value(borrow=True), - "sample_%06i.png" % jj) - tile(rbm.U.get_value(borrow=True).T, "U_%06i.png" % jj) - tile(rbm.W.get_value(borrow=True).T, "W_%06i.png" % jj) - - print 'saving samples', jj, 'epoch', jj / (epoch_size / batchsize) - - print 'l2(U)', l2(rbm.U.get_value(borrow=True)), - print 'l2(W)', l2(rbm.W.get_value(borrow=True)), - print 'l1_penalty', - try: - print trainer.effective_l1_penalty.get_value() - except: - print trainer.effective_l1_penalty - - print 'U min max', rbm.U.get_value(borrow=True).min(), - print rbm.U.get_value(borrow=True).max(), - print 'W min max', rbm.W.get_value(borrow=True).min(), - print rbm.W.get_value(borrow=True).max(), - print 'a min max', rbm.a.get_value(borrow=True).min(), - print rbm.a.get_value(borrow=True).max(), - print 'b min max', rbm.b.get_value(borrow=True).min(), - print rbm.b.get_value(borrow=True).max(), - print 'c min max', rbm.c.get_value(borrow=True).min(), - print rbm.c.get_value(borrow=True).max() - - if persistent_chains: - print 'parts min', smplr.positions.get_value(borrow=True).min(), - print 'max', smplr.positions.get_value(borrow=True).max(), - print 'HMC step', smplr.stepsize.get_value(), - print 'arate', smplr.avg_acceptance_rate.get_value() - - l2_of_Ugrad = learn_fn(jj) - - if persistent_chains and print_jj: - print 'l2(U_grad)', float(l2_of_Ugrad[0]), - print 'l2(U_inc)', float(l2_of_Ugrad[1]), - print 'l2(W_inc)', float(l2_of_Ugrad[2]), - #print 'FE+', float(l2_of_Ugrad[2]), - #print 'FE+[0]', float(l2_of_Ugrad[3]), - #print 'FE+[1]', float(l2_of_Ugrad[4]), - #print 'FE+[2]', float(l2_of_Ugrad[5]), - #print 'FE+[3]', float(l2_of_Ugrad[6]) - - if jj % 2000 == 0: - print '' - print 'Saving rbm...' - cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl' % jj, 'w'), -1) - if persistent_chains: - print 'Saving sampler...' - cPickle.dump(smplr, open('mcRBM.smplr.%06i.pkl' % jj, 'w'), -1) - - return rbm, smplr diff --git a/doc/hmc.txt b/doc/hmc.txt index 0a2a31ec..bf103b10 100644 --- a/doc/hmc.txt +++ b/doc/hmc.txt @@ -10,7 +10,7 @@ Hybrid Monte-Carlo Sampling familiar with Theano and energy-based models such as the RBM. .. note:: - The code for this section is available for download `here `_. + The code for this section is available for download `here `_. Theory @@ -654,11 +654,11 @@ compare the empirical mean and covariance matrix to their true values. assert sampler.stepsize.get_value() <= sampler.stepsize_max -The above code can be run using the command: "nosetests -s code/mcrbm/test\_hmc.py". The output is as follows: +The above code can be run using the command: "nosetests -s code/hmc/test\_hmc.py". The output is as follows: .. code-block:: bash - [desjagui@atchoum mcrbm]$ python test_hmc.py + [desjagui@atchoum hmc]$ python test_hmc.py ****** TARGET VALUES ****** target mean: [ 6.96469186 2.86139335 2.26851454 5.51314769 7.1946897 ] From 240d676668f12ceccd04bd60a46a93f0a0741316 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Fri, 23 Aug 2013 15:09:29 -0400 Subject: [PATCH 006/417] Remove installation of pylearn, and equations for mcrbms --- .travis.yml | 1 - doc/mcrbm.txt | 155 -------------------------------------------------- 2 files changed, 156 deletions(-) delete mode 100644 doc/mcrbm.txt diff --git a/.travis.yml b/.travis.yml index 2a2f88a7..42a93665 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,6 @@ install: #If we don't install numpy before SciPy 0.10.1, the SciPy installations fails. # - "pip install -q scipy --use-mirrors" - "sudo pip install --no-deps git+git://github.com/Theano/Theano.git" - - "sudo pip install hg+http://hg.assembla.com/pylearn" env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp" diff --git a/doc/mcrbm.txt b/doc/mcrbm.txt deleted file mode 100644 index 25a949b4..00000000 --- a/doc/mcrbm.txt +++ /dev/null @@ -1,155 +0,0 @@ -.. _MCRBM: - -Mean Covariance Restricted Boltzmann Machines (mcRBM) -===================================================== - -.. raw:: latex - :label: bigskip - - \bigskip - -Notation -++++++++ - -* :math:`\v \in \mathbb{R}^{D\times 1}`: D Gaussian visible units -* :math:`\h^c \in \{0,1\}^{N\times 1}`: N covariance-hidden units -* :math:`\P \in \mathbb{R}^{F\times N}`: weight matrix connecting N covariance-hidden units to F factors -* :math:`\C \in \mathbb{R}^{D\times F}`: weight matrix connecting F factors to D visible units -* :math:`\b^c \in \mathbb{R}^{N\times 1}`: biases of N covariance-hidden units -* :math:`\h^m \in \{0,1\}^{M\times 1}`: M mean-hidden units -* :math:`\W \in \mathbb{R}^{D\times M}`: weight matrix connecting M mean-hidden units to D visible units -* :math:`\b^m \in \mathbb{R}^{M\times 1}`: biases of M mean-hidden units - -Energy Functions -++++++++++++++++ - -Covariance Energy ------------------ - -.. math:: - :label: cov_energy - - \E^c(\v,\h^c) = -\frac{1}{2} - \sum_{f=1}^F \sum_{k=1}^N P_{fk} h_k^c (\sum_{i=1}^D C_{if} v_i)^2 - - \sum_{k=1}^N b_k^c h_k^c - -Mean Energy ------------ - -.. math:: - :label: mean_energy - - \E^m(\v,\h^m) = - \sum_{j=1}^M \sum_{i=1}^D W_{ij} h_j^m v_i - - \sum_{j=1}^M b_j^m h_j^m - - -Conditionals: :math:`p(\h^m | \v)` ----------------------------------- - -This is the same derivation as with standard RBMs. We start with the observation -that the mean-hidden units are conditionally-independent given the visible -units, hence :math:`p(\h^m | \v) = \prod_{j=1}^M p(\h_j^m | \v)`. - -We can then derive :math:`p(\h_j^m | \v)` as follows: - -.. math:: - - p(\h_j^m | \v) - &= \frac {p(\h_j^m, \v)} {p(\v)} \nonumber \\ - &= \frac {p(\h_j^m, \v)} {\sum_{h_j^m} p(\h_j^m, \v)} \nonumber \\ - &= \frac - {\exp(\sum_{i=1}^D W_{ij} h_j^m v_i + b_j^m h_j^m)} - {\sum_{h_j} \exp(\sum_{i=1}^D W_{ij} h_j^m v_i + b_j^m h_j^m)} \nonumber \\ - &= \frac - {\exp(\sum_{i=1}^D W_{ij} h_j^m v_i + b_j^m h_j^m)} - {1 + \exp(\sum_{i=1}^D W_{ij} v_i + b_j^m)} \nonumber - -The activation probability of a mean-hidden unit, :math:`p(\h_j^m=1 |\v)`, is thus: - -.. math:: - - p(\h_j^m=1 |\v) &= \sigma(\sum_{i=1}^D W_{ij} v_i + b_j^m) \nonumber - - -Conditionals: :math:`p(\h^c | \v)` ----------------------------------- - -It is straightforward to show that the covariance-hidden units are also -conditionally independent. This is due to the fact that :math:`\E^c(\v,\h^c)` is -linear in :math:`\h^c`, thus: - -.. math:: - - p(\h^c, \v) - &= \frac{1}{Z} \exp(-\E^c(\v,\h^c) \nonumber \\ - &= \frac{1}{Z} \exp(\frac{1}{2} \sum_{f=1}^F \sum_{k=1}^N P_{fk} h_k^c (\sum_{i=1}^D C_{if} v_i)^2 + - \sum_{k=1}^N b_k^c h_k^c) \nonumber \\ - &= \frac{1}{Z} \prod_{k=1}^N \exp(\frac{1}{2} \sum_{f=1}^F P_{fk} h_k^c (\sum_{i=1}^D C_{if} v_i)^2 + b_k^c h_k^c) \nonumber \\ - &= \frac{1}{Z} \prod_{k=1}^N p(\h_k^c, \v) \nonumber - -The rest of the derivation is equivalent to \ref{sec:hm_given_v}, substituting -:math:`\E^m(\v,\h^m)` for :math:`\E^c(\v,\h^c)`. This yields: - -.. math:: - - p(\h_k^c=1 |\v) &= \sigma(\frac{1}{2} \sum_{f=1}^F P_{fk} (\sum_{i=1}^D - C_{if} v_i)^2 + b_k^c) \nonumber - - -Conditionals: :math:`p(\v | \h^c,\h^m)` ---------------------------------------- - -From basic probability, we can write: - -.. math:: - - p(\v | \h^c, \h^m) &= \frac {p(\v,\h)} {p(\h)} \nonumber \\ - &= \frac{1}{p(\h)} \frac{1}{Z} \exp( - \frac{1}{2} \sum_{f=1}^F \sum_{k=1}^N P_{fk} h_k^c (\sum_{i=1}^D C_{if} v_i)^2 + \sum_{k=1}^N b_k^c h_k^c + - \sum_{j=1}^M \sum_{i=1}^D W_{ij} h_j^m v_i + \sum_{j=1}^M b_j^m h_j^m) \nonumber \\ - &= \frac{1}{Z_2} \exp( \frac{1}{2} \v^T(\C \text{diag}(\P\h^c) \C^T)\v + {\b^c}^T\h^c + \v^T\W\h^m + {\b^m}^T\h^m) \nonumber - - - - -Setting :math:`\Sigma^{-1} = - \C\text{diag}(P\h^c)\C^T`, we can now write: - -.. math:: - - p(\v | \h^c, \h^m) &= - \frac{1}{Z_2} \exp(-\frac{1}{2} \v^T\Sigma^{-1}\v + {\b^c}^T\h^c + \v^T\W\h^m + {\b^m}^T\h^m) \nonumber \\ - &= \frac{1}{Z_2} \exp(-\frac{1}{2} \v^T\Sigma^{-1}\v + \v\W\h^m) \exp({\b^c}^T\h^c + {\b^m}^T\h^m) \nonumber \\ - &= \frac{1}{Z_3} \exp(-\frac{1}{2} \v^T\Sigma^{-1}\v + \v\W\h^m) \nonumber - -Since we know that :math:`\v` are Gaussian random variables, we need to get -the above in the form :math:`\frac{1}{Z} \exp(-\frac{1}{2} (\v-\mu)^T -\Sigma^{-1} (\v-\mu))`. We can do this by completing the squares and then -solving for :math:`\mu` in the cross-term, which gives -:math:`\v^T \Sigma^{-1} \mu = \v \W \h^m`, and :math:`\mu = \Sigma \W \h^m`. - -Our conditional distribution can thus be written as: - -.. math:: - p(\v | \h^c, \h^m) &= \mathcal{N}(\Sigma \W \h^m, \Sigma) \nonumber \\ - \text{ with } \Sigma^{-1} &= - \C\text{diag}(P\h^c)\C^T \nonumber - - -Free-Energy ------------ - -By definition, the free-energy :math:`\F(\v)` of a given visible configuration :math:`\v` -is: :math:`\F(v) = -\log \sum_h e^{-\E(\v,\h)}`. We can derive the free-energy for -the mcRBM as follows: - -.. math:: - - \F(\v) = &-\log \sum_{h^c} \sum_{h^m} \exp(-\E^c(\v,\h^c) - \E^m(\v,\h^m)) \nonumber \\ - = &-\log \sum_{h^c} \sum_{h^m} \left[ \prod_{k=1}^N \exp(-\E^c(\v,\h_k^c)) - \prod_{j=1}^M \exp(-\E^m(\v,\h_j^m)) \right] \nonumber \\ - = &-\log \left[ \prod_{k=1}^N (1 + \exp(-\E^c(\v,\h_k^c=1))) - \prod_{j=1}^M (1 + \exp(-\E^m(\v,\h_j^m=1))) \right]\nonumber \\ - = &-\sum_{k=1}^N \log(1 + \exp(-\E^c(\v,\h_k^c=1))) - -\sum_{j=1}^M \log(1 + \exp(-\E^m(\v,\h_j^m=1))) \nonumber \\ - = &-\sum_{k=1}^N \log(1 + \exp(\frac{1}{2} \sum_{f=1}^F P_{fk} (\sum_{i=1}^D C_{if} v_i)^2 + b_k^c)) \nonumber \\ - &-\sum_{j=1}^M \log(1 + \exp(\sum_{i=1}^D W_{ij} v_i + b_j^m)) \nonumber - From d2e926db91991cffb33fab53d48a45bae0b16f19 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Fri, 23 Aug 2013 15:09:56 -0400 Subject: [PATCH 007/417] Remove warnings in doc generation --- doc/conf.py | 3 ++- doc/rnnrbm.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index a37a9d97..52631d51 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -111,7 +111,8 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static', 'images'] +#html_static_path = ['.static', 'images'] +html_static_path = ['images'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/rnnrbm.txt b/doc/rnnrbm.txt index ff9d718f..5e7e849e 100644 --- a/doc/rnnrbm.txt +++ b/doc/rnnrbm.txt @@ -400,5 +400,5 @@ The code shown in this tutorial is a stripped-down version that can be improved * Learn the initial condition :math:`u^{(0)}` as a model parameter. -A few samples generated with code including these features are available `here `_. +A few samples generated with code including these features are available here: `sequences.zip `_. From 4c5106ced686b1daee80c7f4ff845b1270073796 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 09:56:53 -0400 Subject: [PATCH 008/417] Download the data in the buildbot script. --- misc/do_nightly_build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/do_nightly_build b/misc/do_nightly_build index 592712e7..bd703f04 100755 --- a/misc/do_nightly_build +++ b/misc/do_nightly_build @@ -8,6 +8,9 @@ NOSETESTS=${ROOT_CWD}/Theano/bin/theano-nose FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} export PYTHONPATH=${ROOT_CWD}/Theano:${ROOT_CWD}/Pylearn:$PYTHONPATH +cd ${ROOT_CWD}/DeepLearningTutorials/data +./download.sh + cd ${ROOT_CWD}/Theano echo "git version for Theano:" `git rev-parse HEAD` cd ${ROOT_CWD}/DeepLearningTutorials/code From 6be486a3da7f24c8e5072547f9a35807eca99f8c Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 09:57:25 -0400 Subject: [PATCH 009/417] Don't redownload the data if it is already downloaded. --- data/download.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/download.sh b/data/download.sh index 237c6ab8..24d6e2b5 100755 --- a/data/download.sh +++ b/data/download.sh @@ -1,5 +1,5 @@ #!/bin/sh -wget http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz -wget http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip Nottingham.zip -wget http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" +wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz +wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip Nottingham.zip +wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From 19b1bec4bb8027a0833f0976938a284ab1536f54 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 10:11:35 -0400 Subject: [PATCH 010/417] Make the unzip always work, that way we can call it from a script. --- data/download.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/download.sh b/data/download.sh index 24d6e2b5..5002f21b 100755 --- a/data/download.sh +++ b/data/download.sh @@ -1,5 +1,5 @@ #!/bin/sh wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz -wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip Nottingham.zip -wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" +wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -f Nottingham.zip +wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -f midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From 3d1a0f9c2e74e2ec053e5ab637ee6990068342e0 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 10:13:21 -0400 Subject: [PATCH 011/417] pep8 --- code/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/test.py b/code/test.py index 63a4f514..59900857 100644 --- a/code/test.py +++ b/code/test.py @@ -50,7 +50,7 @@ def test_dbn(): def test_rbm(): rbm.test_rbm(training_epochs=1, batch_size=300, n_chains=1, n_samples=1, - n_hidden=20, output_folder='tmp_rbm_plots') + n_hidden=20, output_folder='tmp_rbm_plots') def test_rnnrbm(): @@ -65,8 +65,8 @@ def speed(): algo = ['logistic_sgd', 'logistic_cg', 'mlp', 'convolutional_mlp', 'dA', 'SdA', 'DBN', 'rbm', 'rnnrbm'] to_exec = [True] * len(algo) -# to_exec=[False]*len(algo) -# to_exec[-1]=True +# to_exec = [False] * len(algo) +# to_exec[-1] = True do_float64 = True do_float32 = True do_gpu = True From ffca75ad1efb1c02ad433dfb728f1ab1a83e7f90 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 10:27:02 -0400 Subject: [PATCH 012/417] Don't make the path of data files depend of the current directory. --- code/rnnrbm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/rnnrbm.py b/code/rnnrbm.py index f7aad5f9..b55388b8 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -4,6 +4,7 @@ # More information at http://deeplearning.net/tutorial/rnnrbm.html import glob +import os import sys import numpy @@ -269,7 +270,9 @@ def generate(self, filename, show=True): def test_rnnrbm(batch_size=100, num_epochs=200): model = RnnRbm() - model.train(glob.glob('../data/Nottingham/train/*.mid'), + re = os.path.join(os.path.split(os.path.dirname(__file__))[0], + 'data', 'Nottingham', 'train', '*.mid') + model.train(glob.glob(re), batch_size=batch_size, num_epochs=num_epochs) return model From 74599335e9f15f0f9797df454f1f22bf41191bed Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 15:25:04 -0400 Subject: [PATCH 013/417] Add rnnrbm to the travis-ci tests. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42a93665..cf6a4b6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,11 +23,11 @@ install: - "sudo pip install --no-deps git+git://github.com/Theano/Theano.git" env: - - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp" - - PART="test.py:test_convolutional_mlp test.py:test_dA" + - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" - PART="test.py:test_SdA" - PART="test.py:test_dbn" - PART="test.py:test_rbm" + - PART="test.py:test_rnnrbm" - PART="-e test.py" #569.882s #9 code.test.test_rbm OK From 3ea654f191467a3fb4f2dbbfe52330d9803437da Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 12 Sep 2013 15:42:48 -0400 Subject: [PATCH 014/417] Fix the unzip of midi package. --- data/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/download.sh b/data/download.sh index 5002f21b..1cfa7d2a 100755 --- a/data/download.sh +++ b/data/download.sh @@ -2,4 +2,4 @@ wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -f Nottingham.zip -wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -f midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" +wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From 2ac5c4306d2a99ad90a51e255b07b831e43d2ba2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 13 Sep 2013 16:02:26 -0400 Subject: [PATCH 015/417] Add new timming information. --- .travis.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.travis.yml b/.travis.yml index cf6a4b6a..9e71d662 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,40 @@ env: - PART="test.py:test_rnnrbm" - PART="-e test.py" +#i7-2600K CPU @ 3.40GHz +#166.572s #8 test.test_rbm OK +#155.114s #7 test.test_dbn OK +#152.365s #9 test.test_rnnrbm OK +#127.286s #6 test.test_SdA OK +#39.252s #5 test.test_dA OK +#27.56s #4 test.test_convolutional_mlp OK +#15.454s #3 test.test_mlp OK +#12.732s #1 test.test_logistic_sgd OK +#12.638s #2 test.test_logistic_cg OK + +#i7-920 +#296.475s #7 code.test.test_dbn OK +#257.272s #6 code.test.test_SdA OK +#234.776s #9 code.test.test_rnnrbm OK +#233.896s #8 code.test.test_rbm OK +#65.737s #5 code.test.test_dA OK +#37.658s #4 code.test.test_convolutional_mlp OK +#24.172s #3 code.test.test_mlp OK +#20.401s #1 code.test.test_logistic_sgd OK +#17.546s #2 code.test.test_logistic_cg OK + +# On Core2 duo E8500 with MRG +#308.004s #7 code.test.test_dbn OK +#277.268s #6 code.test.test_SdA OK +#126.102s #8 code.test.test_rbm OK +#123.652s #9 code.test.test_rnnrbm OK +#77.101s #5 code.test.test_dA OK +#39.75s #4 code.test.test_convolutional_mlp OK +#30.406s #3 code.test.test_mlp OK +#21.132s #2 code.test.test_logistic_cg OK +#17.945s #1 code.test.test_logistic_sgd OK + +# Unknown computer with older version of Theano #569.882s #9 code.test.test_rbm OK #298.992s #8 code.test.test_dbn OK #268.901s #7 code.test.test_SdA OK From cfb0a0206d6d951157538de3f341283fbe984bea Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 13 Sep 2013 16:19:21 -0400 Subject: [PATCH 016/417] Correctly uncompress the data. --- data/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/download.sh b/data/download.sh index 1cfa7d2a..bc60e638 100755 --- a/data/download.sh +++ b/data/download.sh @@ -1,5 +1,5 @@ #!/bin/sh wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz -wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -f Nottingham.zip +wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From be46753604534c067f1ef18389a98599107fc9c9 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Tue, 8 Oct 2013 17:38:23 -0400 Subject: [PATCH 017/417] Download version of mnist.pkl.gz compatible with python 3 --- data/download.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/data/download.sh b/data/download.sh index bc60e638..ef1b5008 100755 --- a/data/download.sh +++ b/data/download.sh @@ -1,5 +1,6 @@ #!/bin/sh wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz +wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From 3f0b3e237a41da9cb75ce5ef3ee8462a61614a3d Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Tue, 15 Oct 2013 08:15:20 -0400 Subject: [PATCH 018/417] Update timming to lower expected time. --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 59900857..1cf8570b 100644 --- a/code/test.py +++ b/code/test.py @@ -76,7 +76,7 @@ def speed(): # an i7-920 @ 2.67GHz with hyperthread enabled for the cpu # and an GeForce GTX 285 for the GPU. - expected_times_64 = numpy.asarray([10.3, 23.7, 78.1, 73.7, 116.4, + expected_times_64 = numpy.asarray([10.0, 22.5, 76.1, 73.7, 116.4, 346.9, 381.9, 558.1, 186.3]) expected_times_32 = numpy.asarray([11.6, 29.6, 44.2, 66.5, 71, 191.2, 226.8, 432.8, 176.2]) From cac2e23119e2364049da95cedffe761303d638c1 Mon Sep 17 00:00:00 2001 From: emchristiansen Date: Wed, 30 Oct 2013 09:46:53 -0700 Subject: [PATCH 019/417] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 7bcc7474..b3ee5834 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,7 @@ The easiest way to follow the tutorials is to `browse them online `Main development `_ of this project. +[![Build Status](https://travis-ci.org/lisa-lab/DeepLearningTutorials.png)](https://travis-ci.org/lisa-lab/DeepLearningTutorials) Project Layout -------------- From 85b5ddae7619d80e3c91da9bbb7132ad46eac26c Mon Sep 17 00:00:00 2001 From: emchristiansen Date: Wed, 30 Oct 2013 09:51:48 -0700 Subject: [PATCH 020/417] Update README.rst --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b3ee5834..e4a9783d 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,8 @@ The easiest way to follow the tutorials is to `browse them online `Main development `_ of this project. -[![Build Status](https://travis-ci.org/lisa-lab/DeepLearningTutorials.png)](https://travis-ci.org/lisa-lab/DeepLearningTutorials) ++.. image:: https://secure.travis-ci.org/lisa-lab/DeepLearningTutorials.png ++ :target: http://travis-ci.org/lisa-lab/DeepLearningTutorials Project Layout -------------- From 9aed18283d2ea2951249bfc3c5ed1b85a41c9297 Mon Sep 17 00:00:00 2001 From: emchristiansen Date: Wed, 30 Oct 2013 09:52:19 -0700 Subject: [PATCH 021/417] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e4a9783d..85de179c 100644 --- a/README.rst +++ b/README.rst @@ -17,8 +17,8 @@ The easiest way to follow the tutorials is to `browse them online `Main development `_ of this project. -+.. image:: https://secure.travis-ci.org/lisa-lab/DeepLearningTutorials.png -+ :target: http://travis-ci.org/lisa-lab/DeepLearningTutorials +.. image:: https://secure.travis-ci.org/lisa-lab/DeepLearningTutorials.png + :target: http://travis-ci.org/lisa-lab/DeepLearningTutorials Project Layout -------------- From ae246e7215c261b330312c2f0c05f40836cf8428 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 1 Nov 2013 11:31:30 -0400 Subject: [PATCH 022/417] Fix the doc of hmc to use the same code as the code file! This was reported by Chris Fonnesbeck in Theano issue gh-1581. --- doc/hmc.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/hmc.txt b/doc/hmc.txt index bf103b10..2f845509 100644 --- a/doc/hmc.txt +++ b/doc/hmc.txt @@ -649,7 +649,8 @@ compare the empirical mean and covariance matrix to their true values. def test_hmc(): sampler = sampler_on_nd_gaussian(HMC_sampler.new_from_shared_positions, burnin=1000, n_samples=1000, dim=5) - assert abs(sampler.avg_acceptance_rate - sampler.target_acceptance_rate) < .1 + assert abs(sampler.avg_acceptance_rate.get_value() - + sampler.target_acceptance_rate) < .1 assert sampler.stepsize.get_value() >= sampler.stepsize_min assert sampler.stepsize.get_value() <= sampler.stepsize_max From a376f9000459842fc2fee956b64e9fc70708a3d2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 25 Nov 2013 13:46:40 -0500 Subject: [PATCH 023/417] more file to ignore. --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index 512bc4ad..77839e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ code/*.pyc +code/midi data/mnist.pkl.gz +data/mnist_py3k.pkl.gz +data/Nottingham.zip +data/Nottingham +data/midi.zip html *.pyc *~ From c47587036c39793164409987403907dba5d6e9b8 Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 25 Nov 2013 13:48:12 -0500 Subject: [PATCH 024/417] Always load the data from the data folder. This work independently of the current directory. --- code/DBN.py | 2 +- code/SdA.py | 2 +- code/cA.py | 2 +- code/convolutional_mlp.py | 2 +- code/dA.py | 2 +- code/logistic_cg.py | 2 +- code/logistic_sgd.py | 9 ++++++++- code/mlp.py | 2 +- code/rbm.py | 2 +- 9 files changed, 16 insertions(+), 9 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 2289581d..4fdf6c3f 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -257,7 +257,7 @@ def test_score(): def test_DBN(finetune_lr=0.1, pretraining_epochs=100, pretrain_lr=0.01, k=1, training_epochs=1000, - dataset='../data/mnist.pkl.gz', batch_size=10): + dataset='mnist.pkl.gz', batch_size=10): """ Demonstrates how to train and test a Deep Belief Network. diff --git a/code/SdA.py b/code/SdA.py index 71d95463..e000248c 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -295,7 +295,7 @@ def test_score(): def test_SdA(finetune_lr=0.1, pretraining_epochs=15, pretrain_lr=0.001, training_epochs=1000, - dataset='../data/mnist.pkl.gz', batch_size=1): + dataset='mnist.pkl.gz', batch_size=1): """ Demonstrates how to train and test a stochastic denoising autoencoder. diff --git a/code/cA.py b/code/cA.py index 94c5a07e..93ffbf4c 100644 --- a/code/cA.py +++ b/code/cA.py @@ -221,7 +221,7 @@ def get_cost_updates(self, contraction_level, learning_rate): def test_cA(learning_rate=0.01, training_epochs=20, - dataset='../data/mnist.pkl.gz', + dataset='mnist.pkl.gz', batch_size=10, output_folder='cA_plots', contraction_level=.1): """ This demo is tested on MNIST diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 0e32e37e..eba05ac1 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -104,7 +104,7 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): def evaluate_lenet5(learning_rate=0.1, n_epochs=200, - dataset='../data/mnist.pkl.gz', + dataset='mnist.pkl.gz', nkerns=[20, 50], batch_size=500): """ Demonstrates lenet on MNIST dataset diff --git a/code/dA.py b/code/dA.py index c2747a51..e688e72d 100644 --- a/code/dA.py +++ b/code/dA.py @@ -237,7 +237,7 @@ def get_cost_updates(self, corruption_level, learning_rate): def test_dA(learning_rate=0.1, training_epochs=15, - dataset='../data/mnist.pkl.gz', + dataset='mnist.pkl.gz', batch_size=20, output_folder='dA_plots'): """ diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 4d6d65c4..ac227a3a 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -132,7 +132,7 @@ def errors(self, y): raise NotImplementedError() -def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='../data/mnist.pkl.gz'): +def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): """Demonstrate conjugate gradient optimization of a log-linear model This is demonstrated on MNIST. diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 9a164ba7..4bd1a065 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -157,6 +157,13 @@ def load_data(dataset): # Download the MNIST dataset if it is not present data_dir, data_file = os.path.split(dataset) + if data_dir == "" and not os.path.isfile(dataset): + # Check if dataset is in the data directory. + new_path = os.path.join(os.path.split(os.path.split(__file__)[0])[0], + "data", dataset) + if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz': + dataset = new_path + if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz': import urllib origin = 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz' @@ -211,7 +218,7 @@ def shared_dataset(data_xy, borrow=True): def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, - dataset='../data/mnist.pkl.gz', + dataset='mnist.pkl.gz', batch_size=600): """ Demonstrate stochastic gradient descent optimization of a log-linear diff --git a/code/mlp.py b/code/mlp.py index a9c5dc85..629e06bf 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -174,7 +174,7 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, - dataset='../data/mnist.pkl.gz', batch_size=20, n_hidden=500): + dataset='mnist.pkl.gz', batch_size=20, n_hidden=500): """ Demonstrate stochastic gradient descent optimization for a multilayer perceptron diff --git a/code/rbm.py b/code/rbm.py index 1b231652..28608ed9 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -315,7 +315,7 @@ def get_reconstruction_cost(self, updates, pre_sigmoid_nv): def test_rbm(learning_rate=0.1, training_epochs=15, - dataset='../data/mnist.pkl.gz', batch_size=20, + dataset='mnist.pkl.gz', batch_size=20, n_chains=20, n_samples=10, output_folder='rbm_plots', n_hidden=500): """ From 8a9a83d625d00b5f988b103345b5d4c8bb3d0d94 Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 25 Nov 2013 16:00:41 -0500 Subject: [PATCH 025/417] Fix the dataset loading when in the code directory. --- code/logistic_sgd.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 4bd1a065..2fd69c82 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -159,8 +159,12 @@ def load_data(dataset): data_dir, data_file = os.path.split(dataset) if data_dir == "" and not os.path.isfile(dataset): # Check if dataset is in the data directory. - new_path = os.path.join(os.path.split(os.path.split(__file__)[0])[0], - "data", dataset) + new_path = os.path.split(__file__) + if new_path[0] == "": + new_path = os.path.join('..', "data", dataset) + else: + new_path = os.path.join(os.path.split(os.path.split(__file__)[0])[0], + "data", dataset) if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz': dataset = new_path From 42910775f0fedc56ff58b419a8f04716366baa49 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 27 Nov 2013 15:57:21 -0500 Subject: [PATCH 026/417] Remove duplicate code and make it support the new dataset loading fct. --- code/logistic_cg.py | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/code/logistic_cg.py b/code/logistic_cg.py index ac227a3a..dba5d576 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -48,6 +48,8 @@ import theano import theano.tensor as T +from logistic_sgd import load_data + class LogisticRegression(object): """Multi-class Logistic Regression Class @@ -148,41 +150,11 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): ############# # LOAD DATA # ############# - print '... loading data' - - # Load the dataset - f = gzip.open(mnist_pkl_gz, 'rb') - train_set, valid_set, test_set = cPickle.load(f) - f.close() + datasets = load_data(mnist_pkl_gz) - def shared_dataset(data_xy, borrow=True): - """ Function that loads the dataset into shared variables - - The reason we store our dataset in shared variables is to allow - Theano to copy it into the GPU memory (when code is run on GPU). - Since copying data into the GPU is slow, copying a minibatch everytime - is needed (the default behaviour if the data is not in a shared - variable) would lead to a large decrease in performance. - """ - data_x, data_y = data_xy - shared_x = theano.shared(numpy.asarray(data_x, - dtype=theano.config.floatX), - borrow=borrow) - shared_y = theano.shared(numpy.asarray(data_y, - dtype=theano.config.floatX), - borrow=borrow) - # When storing data on the GPU it has to be stored as floats - # therefore we will store the labels as ``floatX`` as well - # (``shared_y`` does exactly that). But during our computations - # we need them as ints (we use labels as index, and if they are - # floats it doesn't make sense) therefore instead of returning - # ``shared_y`` we will have to cast it to int. This little hack - # lets ous get around this issue - return shared_x, T.cast(shared_y, 'int32') - - test_set_x, test_set_y = shared_dataset(test_set) - valid_set_x, valid_set_y = shared_dataset(valid_set) - train_set_x, train_set_y = shared_dataset(train_set) + train_set_x, train_set_y = datasets[0] + valid_set_x, valid_set_y = datasets[1] + test_set_x, test_set_y = datasets[2] batch_size = 600 # size of the minibatch From 4271d5d693b8fe537600895ae062e0e0c15e923f Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 27 Nov 2013 16:02:32 -0500 Subject: [PATCH 027/417] make exactly 5 part as travis-ci run only 5 jobs in parallel. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e71d662..ae499e46 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,7 @@ env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" - PART="test.py:test_SdA" - PART="test.py:test_dbn" - - PART="test.py:test_rbm" - - PART="test.py:test_rnnrbm" + - PART="test.py:test_rbm test.py:test_rnnrbm" - PART="-e test.py" #i7-2600K CPU @ 3.40GHz From 6069c87b0e947ae32250cce8b48898709d347561 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 28 Nov 2013 09:28:22 -0500 Subject: [PATCH 028/417] Code simplication following code review. --- code/logistic_sgd.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 2fd69c82..074d4c13 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -159,12 +159,7 @@ def load_data(dataset): data_dir, data_file = os.path.split(dataset) if data_dir == "" and not os.path.isfile(dataset): # Check if dataset is in the data directory. - new_path = os.path.split(__file__) - if new_path[0] == "": - new_path = os.path.join('..', "data", dataset) - else: - new_path = os.path.join(os.path.split(os.path.split(__file__)[0])[0], - "data", dataset) + new_path = os.path.join(os.path.split(__file__)[0], "..", "data", dataset) if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz': dataset = new_path From d71526094d35de0cec9b0dbad66678829213ba8d Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Mon, 30 Dec 2013 13:12:57 -0500 Subject: [PATCH 029/417] Change equation formatting to code formatting build_finetune_functions is being rendered as a broken math equation, not a code snippet. --- doc/SdA.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/SdA.txt b/doc/SdA.txt index 4f626ec4..3aa0d8bb 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -410,7 +410,7 @@ to the training set for a fixed number of epochs given by The fine-tuning loop is very similar with the one in the :ref:`mlp`, the only difference is that we will use now the functions given by -`build_finetune_functions` . +``build_finetune_functions`` . From a376f41de71987edf538ef7fed4118ca173f65a7 Mon Sep 17 00:00:00 2001 From: Zac Stewart Date: Mon, 30 Dec 2013 13:52:08 -0500 Subject: [PATCH 030/417] Change references to SigmoidalLayer to HiddenLayer The SigmoidalLayer and TahnLayer classes seem to have been merged into the HiddenLayer with has an activation function attribute. Several places in the tutorial still refer to the SigmoidalLayer, so I updated them to HiddenLayer and took care to specify the T.nnet.sigmoid activation function where appropriate. --- code/mlp.py | 10 +++++----- doc/DBN.txt | 2 +- doc/SdA.txt | 7 ++++--- doc/lenet.txt | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/code/mlp.py b/code/mlp.py index 629e06bf..8ac1d496 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -108,7 +108,7 @@ class MLP(object): A multilayer perceptron is a feedforward artificial neural network model that has one layer or more of hidden units and nonlinear activations. Intermediate layers usually have as activation function thanh or the - sigmoid function (defined here by a ``SigmoidalLayer`` class) while the + sigmoid function (defined here by a ``HiddenLayer`` class) while the top layer is a softamx layer (defined here by a ``LogisticRegression`` class). """ @@ -136,10 +136,10 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): """ - # Since we are dealing with a one hidden layer MLP, this will - # translate into a TanhLayer connected to the LogisticRegression - # layer; this can be replaced by a SigmoidalLayer, or a layer - # implementing any other nonlinearity + # Since we are dealing with a one hidden layer MLP, this will translate + # into a HiddenLayer with a tanh activation function connected to the + # LogisticRegression layer; the activation function can be replaced by + # sigmoid or any other nonlinear function self.hiddenLayer = HiddenLayer(rng=rng, input=input, n_in=n_in, n_out=n_hidden, activation=T.tanh) diff --git a/doc/DBN.txt b/doc/DBN.txt index f2db5e4b..eeea3602 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -189,7 +189,7 @@ the MLP, while ``self.rbm_layers`` will store the RBMs used to pretrain each layer of the MLP. Next step, we construct ``n_layers`` sigmoid layers (we use the -``SigmoidalLayer`` class introduced in :ref:`mlp`, with the only modification +``HiddenLayer`` class introduced in :ref:`mlp`, with the only modification that we replaced the non-linearity from ``tanh`` to the logistic function :math:`s(x) = \frac{1}{1+e^{-x}}`) and ``n_layers`` RBMs, where ``n_layers`` is the depth of our model. We link the sigmoid layers such that they form an diff --git a/doc/SdA.txt b/doc/SdA.txt index 3aa0d8bb..02b49d6c 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -126,7 +126,7 @@ representations of intermediate layers of the MLP. ``self.dA_layers`` will store the denoising autoencoder associated with the layers of the MLP. Next step, we construct ``n_layers`` sigmoid layers (we use the -``SigmoidalLayer`` class introduced in :ref:`mlp`, with the only +``HiddenLayer`` class introduced in :ref:`mlp`, with the only modification that we replaced the non-linearity from ``tanh`` to the logistic function :math:`s(x) = \frac{1}{1+e^{-x}}`) and ``n_layers`` denoising autoencoders, where ``n_layers`` is the depth of our model. @@ -154,10 +154,11 @@ bias of the encoding part with its corresponding sigmoid layer. else: layer_input = self.sigmoid_layers[-1].output - sigmoid_layer = SigmoidalLayer(rng=rng, + sigmoid_layer = HiddenLayer(rng=rng, input=layer_input, n_in=input_size, - n_out=hidden_layers_sizes[i]) + n_out=hidden_layers_sizes[i], + activation=T.nnet.sigmoid) # add the layer to our list of layers self.sigmoid_layers.append(sigmoid_layer) diff --git a/doc/lenet.txt b/doc/lenet.txt index e479abe7..ffd36a19 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -498,7 +498,7 @@ instantiate the network as follows. image_shape=(batch_size, 20, 12, 12), filter_shape=(50, 20, 5, 5), poolsize=(2, 2)) - # the SigmoidalLayer being fully-connected, it operates on 2D matrices of + # the HiddenLayer being fully-connected, it operates on 2D matrices of # shape (batch_size,num_pixels) (i.e matrix of rasterized images). # This will generate a matrix of shape (20, 32 * 4 * 4) = (20, 512) layer2_input = layer1.output.flatten(2) From 00961d2d02155640f8fc1f315efc6561052e60bd Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 6 Jan 2014 10:15:59 -0500 Subject: [PATCH 031/417] Fix typo and fix reference to old class that was renamed --- code/convolutional_mlp.py | 2 +- code/mlp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index eba05ac1..a53386cd 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -171,7 +171,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, image_shape=(batch_size, nkerns[0], 12, 12), filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2)) - # the TanhLayer being fully-connected, it operates on 2D matrices of + # the HiddenLayer being fully-connected, it operates on 2D matrices of # shape (batch_size,num_pixels) (i.e matrix of rasterized images). # This will generate a matrix of shape (20,32*4*4) = (20,512) layer2_input = layer1.output.flatten(2) diff --git a/code/mlp.py b/code/mlp.py index 8ac1d496..05c227b0 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -107,7 +107,7 @@ class MLP(object): A multilayer perceptron is a feedforward artificial neural network model that has one layer or more of hidden units and nonlinear activations. - Intermediate layers usually have as activation function thanh or the + Intermediate layers usually have as activation function tanh or the sigmoid function (defined here by a ``HiddenLayer`` class) while the top layer is a softamx layer (defined here by a ``LogisticRegression`` class). From c53165db183f882aad81d0e516f70e36bff54809 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 24 Jan 2014 15:15:15 -0500 Subject: [PATCH 032/417] Added info on the computer who do the speed test. --- code/test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/test.py b/code/test.py index 1cf8570b..99ed13d0 100644 --- a/code/test.py +++ b/code/test.py @@ -72,9 +72,10 @@ def speed(): do_gpu = True algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]] - #Timming expected are from the buildbot that have - # an i7-920 @ 2.67GHz with hyperthread enabled for the cpu - # and an GeForce GTX 285 for the GPU. + #Timming expected are from the buildbot that have an i7-920 @ + # 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX + # 285 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD + # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([10.0, 22.5, 76.1, 73.7, 116.4, 346.9, 381.9, 558.1, 186.3]) From 5d184e4102b9f56d7b611d6c6d2b81dcb9d250d0 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 7 Feb 2014 13:04:37 -0500 Subject: [PATCH 033/417] A speed test was failing as we are faster now! So update the expected run time. --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 99ed13d0..28697613 100644 --- a/code/test.py +++ b/code/test.py @@ -79,7 +79,7 @@ def speed(): expected_times_64 = numpy.asarray([10.0, 22.5, 76.1, 73.7, 116.4, 346.9, 381.9, 558.1, 186.3]) - expected_times_32 = numpy.asarray([11.6, 29.6, 44.2, 66.5, 71, + expected_times_32 = numpy.asarray([11.6, 29.6, 42.5, 66.5, 71, 191.2, 226.8, 432.8, 176.2]) # Number with just 1 decimal are new value that are faster with From fc006cc13e5c59eff4c13bc3a2bd188699faa24b Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 20 Feb 2014 12:51:07 -0500 Subject: [PATCH 034/417] Allow using curl to download if wget is not installed. --- data/download.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/data/download.sh b/data/download.sh index ef1b5008..2f1cf77d 100755 --- a/data/download.sh +++ b/data/download.sh @@ -1,6 +1,19 @@ #!/bin/sh -wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz -wget -c http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz -wget -c http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip -wget -c http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" +which wget >/dev/null 2>&1 +WGET=$? +which curl >/dev/null 2>&1 +CURL=$? +if [ "$WGET" -eq 0 ]; then + DL_CMD="wget -c" +elif [ "$CURL" -eq 0 ]; then + DL_CMD="curl -C - -O" +else + echo "You need wget or curl installed to download" + exit 1 +fi + +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" From f6505b69a9f093c3d47fc9518935958ccef6d317 Mon Sep 17 00:00:00 2001 From: Markus Roth Date: Sun, 16 Mar 2014 16:46:16 +0100 Subject: [PATCH 035/417] Fix typo. --- doc/rbm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rbm.txt b/doc/rbm.txt index 21b3e0a7..d0c7e1ae 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -573,7 +573,7 @@ chain to get the free energy of the negative phase. Note that the ``chain_end`` is a symbolical Theano variable expressed in terms of the model parameters, and if we would apply ``T.grad`` naively, the function will try to go through the Gibbs chain to get the gradients. This is not what we -want (it will mess up our gradients) and therefire we need to indicate to +want (it will mess up our gradients) and therefore we need to indicate to ``T.grad`` that ``chain_end`` is a constant. We do this by using the argument ``consider_constant`` of ``T.grad``. From fcac646ce2a89412e06037da5581cf42b8f95dcd Mon Sep 17 00:00:00 2001 From: Markus Roth Date: Sun, 16 Mar 2014 16:52:01 +0100 Subject: [PATCH 036/417] Fix typo. --- doc/rbm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rbm.txt b/doc/rbm.txt index d0c7e1ae..d86da47f 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -651,7 +651,7 @@ all bits are independent. Therefore, Here :math:`x_{-i}` denotes the set of all bits of :math:`x` except bit :math:`i`. The log-PL is therefore the sum of the log-probabilities of each -bit :math:`x_i`, conditionned on the state of all other bits. For MNIST, this +bit :math:`x_i`, conditioned on the state of all other bits. For MNIST, this would involve summing over the 784 input dimensions, which remains rather expensive. For this reason, we use the following stochastic approximation to log-PL: From 775c76b9f266abf19098d42acfae11cf0d723d8f Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Apr 2014 11:47:04 -0400 Subject: [PATCH 037/417] Add link to the 3wolfmoon image and fix its path --- doc/lenet.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index ffd36a19..85b98550 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -42,10 +42,12 @@ Convolutional Neural Networks (LeNet) .. _dimshuffle: http://deeplearning.net/software/theano/library/tensor/basic.html#tensor._tensor_py_operators.dimshuffle .. note:: - The code for this section is available for download `here`_. + The code for this section is available for download `here`_ and the `3wolfmoon image`_ .. _here: http://deeplearning.net/tutorial/code/convolutional_mlp.py +.. _3wolfmoon image: https://raw.githubusercontent.com/lisa-lab/DeepLearningTutorials/master/doc/images/3wolfmoon.jpg + Motivation ++++++++++ @@ -258,7 +260,7 @@ Let's have a little bit of fun with this... from PIL import Image # open random image of dimensions 639x516 - img = Image.open(open('images/3wolfmoon.jpg')) + img = Image.open(open('doc/images/3wolfmoon.jpg')) img = numpy.asarray(img, dtype='float64') / 256. # put image in 4D tensor of shape (1, 3, height, width) From f33b4374a777745f058abfbf3196a78e5c156aec Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Apr 2014 11:48:17 -0400 Subject: [PATCH 038/417] Update comment about Theano version. --- doc/rnnrbm.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/rnnrbm.txt b/doc/rnnrbm.txt index 5e7e849e..ea95e177 100644 --- a/doc/rnnrbm.txt +++ b/doc/rnnrbm.txt @@ -20,8 +20,7 @@ Modeling and generating sequences of polyphonic music with the RNN-RBM Note that both dependencies above can be setup automatically by running the ``download.sh`` script in the ``../data`` directory. .. caution:: - Depending on your locally installed Theano version, you may have problems running this script. - If this is the case, please use the `'bleeding-edge' developer version `_ from github. + Need Theano 0.6 or more recent. The RNN-RBM From e7f039707ea7f77b2a681d98ccc4dc68d663b941 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 7 May 2014 11:07:11 -0400 Subject: [PATCH 039/417] fix a comment and give more info about it. --- doc/logreg.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/logreg.txt b/doc/logreg.txt index 3bcc0559..cd85f252 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -68,8 +68,11 @@ The code to do this in Theano is the following: b = theano.shared(numpy.zeros((10,)), name='b') W = theano.shared(numpy.zeros((784, 10)), name='W') - # symbolic expression for computing the vector of - # class-membership probabilities + # symbolic expression for computing the matrix of class-membership probabilities + # Where: + # W is a matrix where column-k represent the separation hyper plain for class-k + # x is a matrix where row-j represents input training sample-j + # b is a vector where element-k represent the free parameter of hyper plain-k p_y_given_x = T.nnet.softmax(T.dot(x, W) + b) # compiled Theano function that returns the vector of class-membership From ae371140536d9f8ab7072384c786933db1711dc7 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 22 May 2014 11:14:58 -0400 Subject: [PATCH 040/417] back-port fix to the doc. --- doc/lenet.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 85b98550..a4eb86fa 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -473,8 +473,8 @@ instantiate the network as follows. batch_size = 20 # sized of the minibatch # allocate symbolic variables for the data - x = theano.floatX.xmatrix(theano.config.floatX) # rasterized images - y = T.lvector() # the labels are presented as 1D vector of [long int] labels + x = T.matrix('x') # rasterized images + y = T.lvector('y') # the labels are presented as 1D vector of [long int] labels ############################## # BEGIN BUILDING ACTUAL MODE From b88e1d3b6940109d647cad939c830ef3c8fb049b Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 11 Jun 2014 11:58:14 -0400 Subject: [PATCH 041/417] date timming to those from a 580 as the buildbot GPU changed. --- code/test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/code/test.py b/code/test.py index 28697613..d1eedb5a 100644 --- a/code/test.py +++ b/code/test.py @@ -74,7 +74,7 @@ def speed(): algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]] #Timming expected are from the buildbot that have an i7-920 @ # 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX - # 285 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD + # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([10.0, 22.5, 76.1, 73.7, 116.4, @@ -87,7 +87,7 @@ def speed(): # updated, as we where faster in the past! # TODO: find why and fix this! -# Here is the value for the buildbot on February 3th 2012. +# Here is the value for the buildbot on February 3th 2012 with a GTX 285 # sgd, cg mlp conv da # sda dbn rbm # gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657, @@ -98,9 +98,11 @@ def speed(): # sda dbn rbm #expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138 # 1.35324519 1.7356905 1.12937868] - expected_times_gpu = numpy.asarray([3.07663488, 7.55523491, 18.99226785, - 9.6, 24.13007045, - 20.4, 56, 302.6, 315.4]) + + expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, + 5.8, 21.5, + 11.8, 47.9, 290.1, 315.4]) + expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 00e98ff92d743d35f83434af006148af9559abdb Mon Sep 17 00:00:00 2001 From: Renaud Richardet Date: Thu, 7 Aug 2014 10:57:53 +0200 Subject: [PATCH 042/417] Update DBN.py minor fix on the documentation of field names --- code/DBN.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 4fdf6c3f..4a287dd4 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -43,8 +43,8 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, :type n_ins: int :param n_ins: dimension of the input to the DBN - :type n_layers_sizes: list of ints - :param n_layers_sizes: intermediate layers size, must contain + :type hidden_layers_sizes: list of ints + :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int @@ -263,8 +263,8 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, This is demonstrated on MNIST. - :type learning_rate: float - :param learning_rate: learning rate used in the finetune stage + :type finetune_lr: float + :param finetune_lr: learning rate used in the finetune stage :type pretraining_epochs: int :param pretraining_epochs: number of epoch to do pretraining :type pretrain_lr: float From 5a0aa644ca1b9c1717df0780ecae82f5f369a4cd Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 7 Aug 2014 09:45:52 -0400 Subject: [PATCH 043/417] Do the same fix in the doc then what was done in the code. --- doc/DBN.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/DBN.txt b/doc/DBN.txt index eeea3602..b2e5230a 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -161,8 +161,8 @@ classification. :type n_ins: int :param n_ins: dimension of the input to the DBN - :type n_layers_sizes: list of ints - :param n_layers_sizes: intermediate layers size, must contain + :type hidden_layers_sizes: list of ints + :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int From 93de4ed3ce6d5e6289cde054eb227da79f4fbf4b Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 2 Oct 2014 09:28:30 -0400 Subject: [PATCH 044/417] Fix the math in lenet --- doc/lenet.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index a4eb86fa..3035115a 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -139,10 +139,10 @@ feature map :math:`h^k` is obtained as follows (for :math:`tanh` non-linearities .. Note:: Recall the following definition of convolution for a 1D signal. - :math:`o[n] = f[n]*g[n] = \sum_{u=-\infty}^{\infty} f[u] g[u-n] = \sum_{u=-\infty}^{\infty} f[n-u] g[u]`. + :math:`o[n] = f[n]*g[n] = \sum_{u=-\infty}^{\infty} f[u] g[n-u] = \sum_{u=-\infty}^{\infty} f[n-u] g[u]`. This can be extended to 2D as follows: - :math:`o[m,n] = f[m,n]*g[m,n] = \sum_{u=-\infty}^{\infty} \sum_{v=-\infty}^{\infty} f[u,v] g[u-m,v-n]`. + :math:`o[m,n] = f[m,n]*g[m,n] = \sum_{u=-\infty}^{\infty} \sum_{v=-\infty}^{\infty} f[u,v] g[m-u,n-v]`. To form a richer representation of the data, hidden layers are composed of a set of multiple feature maps, :math:`\{h^{(k)}, k=0..K\}`. From fef605709b5bf9f7c0c7452cdf52ef1f916f387a Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 2 Oct 2014 09:29:57 -0400 Subject: [PATCH 045/417] Update speed timing to faster results. --- code/test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/test.py b/code/test.py index d1eedb5a..5cb13c89 100644 --- a/code/test.py +++ b/code/test.py @@ -77,9 +77,9 @@ def speed(): # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. - expected_times_64 = numpy.asarray([10.0, 22.5, 76.1, 73.7, 116.4, + expected_times_64 = numpy.asarray([9.8, 22.5, 76.1, 73.7, 116.4, 346.9, 381.9, 558.1, 186.3]) - expected_times_32 = numpy.asarray([11.6, 29.6, 42.5, 66.5, 71, + expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, 191.2, 226.8, 432.8, 176.2]) # Number with just 1 decimal are new value that are faster with @@ -219,6 +219,7 @@ def do_tests(): print >> sys.stderr, 'gpu % expected/get', ( expected_times_gpu / gpu_times) + print if do_float64 and do_float32: print >> sys.stderr, 'float64/float32', ( float64_times / float32_times) @@ -239,6 +240,7 @@ def compare(x, y): # time and the real time, we consider this an error. return sum((ratio < 0.95) + (ratio > 1.05)) + print if do_float64: err = compare(expected_times_64, float64_times) print >> sys.stderr, 'speed_failure_float64=' + str(err) From 15a2cefdbef9016ff6301406fdd06c19f88da360 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Tue, 7 Oct 2014 19:43:42 -0700 Subject: [PATCH 046/417] Alternative to PIL.image for Mac OS X. --- code/cA.py | 7 +++++-- code/dA.py | 9 ++++++--- code/rbm.py | 10 +++++++--- code/utils.py | 2 +- doc/dA.txt | 2 +- doc/rbm.txt | 4 ++-- doc/utilities.txt | 2 +- 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/code/cA.py b/code/cA.py index 93ffbf4c..986a2d2a 100644 --- a/code/cA.py +++ b/code/cA.py @@ -42,7 +42,10 @@ from logistic_sgd import load_data from utils import tile_raster_images -import PIL.Image +try: + import PIL.Image as Image +except ImportError: + import Image class cA(object): @@ -290,7 +293,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) - image = PIL.Image.fromarray(tile_raster_images( + image = Image.fromarray(tile_raster_images( X=ca.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) diff --git a/code/dA.py b/code/dA.py index e688e72d..86b7fe86 100644 --- a/code/dA.py +++ b/code/dA.py @@ -45,7 +45,10 @@ from logistic_sgd import load_data from utils import tile_raster_images -import PIL.Image +try: + import PIL.Image as Image +except ImportError: + import Image class dA(object): @@ -306,7 +309,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, print >> sys.stderr, ('The no corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((training_time) / 60.)) - image = PIL.Image.fromarray( + image = Image.fromarray( tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) @@ -352,7 +355,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, os.path.split(__file__)[1] + ' ran for %.2fm' % (training_time / 60.)) - image = PIL.Image.fromarray(tile_raster_images( + image = Image.fromarray(tile_raster_images( X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) diff --git a/code/rbm.py b/code/rbm.py index 28608ed9..a3013b41 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -7,7 +7,11 @@ import cPickle import gzip import time -import PIL.Image + +try: + import PIL.Image as Image +except ImportError: + import Image import numpy @@ -396,7 +400,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, # Plot filters after each training epoch plotting_start = time.clock() # Construct image from the weight matrix - image = PIL.Image.fromarray(tile_raster_images( + image = Image.fromarray(tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) @@ -459,7 +463,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, tile_spacing=(1, 1)) # construct image - image = PIL.Image.fromarray(image_data) + image = Image.fromarray(image_data) image.save('samples.png') os.chdir('../') diff --git a/code/utils.py b/code/utils.py index 9261960a..6313ed2f 100644 --- a/code/utils.py +++ b/code/utils.py @@ -47,7 +47,7 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), :returns: array suitable for viewing as an image. - (See:`PIL.Image.fromarray`.) + (See:`Image.fromarray`.) :rtype: a 2-d array with same dtype as X. """ diff --git a/doc/dA.txt b/doc/dA.txt index ee27f1d3..f97140f1 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -588,7 +588,7 @@ save the filters as an image : .. code-block:: python - image = PIL.Image.fromarray(tile_raster_images(X=da.W.get_value(borrow=True).T, + image = Image.fromarray(tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_30.png') diff --git a/doc/rbm.txt b/doc/rbm.txt index d86da47f..67f99e5a 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -754,7 +754,7 @@ been shown to lead to a better generative model ([Tieleman08]_). # Plot filters after each training epoch plotting_start = time.clock() # Construct image from the weight matrix - image = PIL.Image.fromarray(tile_raster_images( + image = Image.fromarray(tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) @@ -841,7 +841,7 @@ samples at every 1000 steps. tile_spacing=(1, 1)) # construct image - image = PIL.Image.fromarray(image_data) + image = Image.fromarray(image_data) print ' ... plotting sample ', idx image.save('samples.png') diff --git a/doc/utilities.txt b/doc/utilities.txt index 9f3f9dae..0367127c 100644 --- a/doc/utilities.txt +++ b/doc/utilities.txt @@ -78,7 +78,7 @@ Tiling minibatches together is done for us by the :returns: array suitable for viewing as an image. - (See:`PIL.Image.fromarray`.) + (See:`Image.fromarray`.) :rtype: a 2-d array with same dtype as X. """ From 0683244c8ea5364be51ef08db8cbc27a505caa0d Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 15:06:27 -0700 Subject: [PATCH 047/417] In the ReStructuredText (*.txt) files, many of the copies of Python code were out of date. Replaced copies of Python code with links to sections of the corresponding files. This dramatically reduced the number of lines of code to maintain. This will help keep documentation in sync with code (but will not guarantee synchronization). Renamed two ReStructuredText files to match corresponding Python files. Modified code files for improved readability. Modified .gitignore to ignore tmp directories. --- doc/{lenet.txt => convolutional_mlp.txt} | 0 doc/{logreg.txt => logistic_sgd.txt} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename doc/{lenet.txt => convolutional_mlp.txt} (100%) rename doc/{logreg.txt => logistic_sgd.txt} (100%) diff --git a/doc/lenet.txt b/doc/convolutional_mlp.txt similarity index 100% rename from doc/lenet.txt rename to doc/convolutional_mlp.txt diff --git a/doc/logreg.txt b/doc/logistic_sgd.txt similarity index 100% rename from doc/logreg.txt rename to doc/logistic_sgd.txt From 1d8b9fdc35f1d777e38bb8547c5a444dd4b81b30 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 15:33:31 -0700 Subject: [PATCH 048/417] In the ReStructuredText (*.txt) files, many of the copies of Python code were out of date. Replaced copies of Python code with links to sections of the corresponding files. This dramatically reduced the number of lines of code to maintain. This will help keep documentation in sync with code (but will not guarantee synchronization). Renamed two ReStructuredText files to match corresponding Python files. Modified code files for improved readability. Modified .gitignore to ignore tmp directories. --- .gitignore | 1 + code/SdA.py | 88 +++--- code/convolutional_mlp.py | 111 ++++--- code/dA.py | 68 +++-- code/logistic_sgd.py | 93 +++--- code/mlp.py | 77 +++-- code/rbm.py | 51 +++- code/rnnrbm.py | 182 +++++------ doc/DBN.txt | 282 ++--------------- doc/SdA.txt | 301 ++---------------- doc/contents.txt | 4 +- doc/convolutional_mlp.txt | 144 +-------- doc/dA.txt | 136 +-------- doc/hmc.txt | 402 ++----------------------- doc/intro.txt | 2 +- doc/logistic_sgd.txt | 194 ++---------- doc/mlp.txt | 207 ++----------- doc/rbm.txt | 398 +++--------------------- doc/rnnrbm.txt | 249 +-------------- issues_open/6_benchmarking_pybrain.txt | 2 +- 20 files changed, 602 insertions(+), 2390 deletions(-) diff --git a/.gitignore b/.gitignore index 77839e2e..3392e83c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ code/*.pyc +code/tmp* code/midi data/mnist.pkl.gz data/mnist_py3k.pkl.gz diff --git a/code/SdA.py b/code/SdA.py index e000248c..26927d3d 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -57,9 +57,14 @@ class SdA(object): the dAs are only used to initialize the weights. """ - def __init__(self, numpy_rng, theano_rng=None, n_ins=784, - hidden_layers_sizes=[500, 500], n_outs=10, - corruption_levels=[0.1, 0.1]): + def __init__(self, + numpy_rng, + theano_rng = None, + n_ins = 784, + hidden_layers_sizes = [500, 500], + n_outs = 10, + corruption_levels = [0.1, 0.1] + ): """ This class is made to support a variable number of layers. :type numpy_rng: numpy.random.RandomState @@ -241,9 +246,9 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): (test_set_x, test_set_y) = datasets[2] # compute number of minibatches for training, validation and testing - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] + n_valid_batches = valid_set_x.get_value(borrow = True).shape[0] n_valid_batches /= batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] + n_test_batches = test_set_x.get_value(borrow = True).shape[0] n_test_batches /= batch_size index = T.lscalar('index') # index to a [mini]batch @@ -252,35 +257,41 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): gparams = T.grad(self.finetune_cost, self.params) # compute list of fine-tuning updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - gparam * learning_rate)) - - train_fn = theano.function(inputs=[index], - outputs=self.finetune_cost, - updates=updates, - givens={ - self.x: train_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: train_set_y[index * batch_size: - (index + 1) * batch_size]}, - name='train') - - test_score_i = theano.function([index], self.errors, - givens={ - self.x: test_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: test_set_y[index * batch_size: - (index + 1) * batch_size]}, - name='test') - - valid_score_i = theano.function([index], self.errors, - givens={ - self.x: valid_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: valid_set_y[index * batch_size: - (index + 1) * batch_size]}, - name='valid') + updates = [ + (param, param - gparam * learning_rate) + for param, gparam in zip(self.params, gparams) + ] + + train_fn = theano.function( + inputs = [index], + outputs = self.finetune_cost, + updates = updates, + givens = { + self.x: train_set_x[index * batch_size : (index + 1) * batch_size], + self.y: train_set_y[index * batch_size : (index + 1) * batch_size] + }, + name = 'train' + ) + + test_score_i = theano.function( + [index], + self.errors, + givens = { + self.x: test_set_x[index * batch_size : (index + 1) * batch_size], + self.y: test_set_y[index * batch_size : (index + 1) * batch_size] + }, + name = 'test' + ) + + valid_score_i = theano.function( + [index], + self.errors, + givens = { + self.x: valid_set_x[index * batch_size : (index + 1) * batch_size], + self.y: valid_set_y[index * batch_size : (index + 1) * batch_size] + }, + name='valid' + ) # Create a function that scans the entire validation set def valid_score(): @@ -333,9 +344,12 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, numpy_rng = numpy.random.RandomState(89677) print '... building the model' # construct the stacked denoising autoencoder class - sda = SdA(numpy_rng=numpy_rng, n_ins=28 * 28, - hidden_layers_sizes=[1000, 1000, 1000], - n_outs=10) + sda = SdA( + numpy_rng = numpy_rng, + n_ins = 28 * 28, + hidden_layers_sizes = [1000, 1000, 1000], + n_outs = 10 + ) ######################### # PRETRAINING THE MODEL # diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index a53386cd..dbcbfa69 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -76,22 +76,32 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) - self.W = theano.shared(numpy.asarray( - rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), - dtype=theano.config.floatX), - borrow=True) + self.W = theano.shared( + numpy.asarray( + rng.uniform(low = -W_bound, high = W_bound, size = filter_shape), + dtype = theano.config.floatX + ), + borrow = True + ) # the bias is a 1D tensor -- one bias per output feature map b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) # convolve input feature maps with filters - conv_out = conv.conv2d(input=input, filters=self.W, - filter_shape=filter_shape, image_shape=image_shape) + conv_out = conv.conv2d( + input = input, + filters = self.W, + filter_shape = filter_shape, + image_shape = image_shape + ) # downsample each feature map individually, using maxpooling - pooled_out = downsample.max_pool_2d(input=conv_out, - ds=poolsize, ignore_border=True) + pooled_out = downsample.max_pool_2d( + input = conv_out, + ds = poolsize, + ignore_border = True + ) # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will @@ -131,9 +141,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] - n_test_batches = test_set_x.get_value(borrow=True).shape[0] + n_train_batches = train_set_x.get_value(borrow = True).shape[0] + n_valid_batches = valid_set_x.get_value(borrow = True).shape[0] + n_test_batches = test_set_x.get_value(borrow = True).shape[0] n_train_batches /= batch_size n_valid_batches /= batch_size n_test_batches /= batch_size @@ -159,17 +169,25 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # filtering reduces the image size to (28-5+1,28-5+1)=(24,24) # maxpooling reduces this further to (24/2,24/2) = (12,12) # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12) - layer0 = LeNetConvPoolLayer(rng, input=layer0_input, - image_shape=(batch_size, 1, 28, 28), - filter_shape=(nkerns[0], 1, 5, 5), poolsize=(2, 2)) + layer0 = LeNetConvPoolLayer( + rng, + input = layer0_input, + image_shape = (batch_size, 1, 28, 28), + filter_shape = (nkerns[0], 1, 5, 5), + poolsize = (2, 2) + ) # Construct the second convolutional pooling layer # filtering reduces the image size to (12-5+1,12-5+1)=(8,8) # maxpooling reduces this further to (8/2,8/2) = (4,4) # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4) - layer1 = LeNetConvPoolLayer(rng, input=layer0.output, - image_shape=(batch_size, nkerns[0], 12, 12), - filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2)) + layer1 = LeNetConvPoolLayer( + rng, + input = layer0.output, + image_shape = (batch_size, nkerns[0], 12, 12), + filter_shape = (nkerns[1], nkerns[0], 5, 5), + poolsize = (2, 2) + ) # the HiddenLayer being fully-connected, it operates on 2D matrices of # shape (batch_size,num_pixels) (i.e matrix of rasterized images). @@ -177,25 +195,38 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, layer2_input = layer1.output.flatten(2) # construct a fully-connected sigmoidal layer - layer2 = HiddenLayer(rng, input=layer2_input, n_in=nkerns[1] * 4 * 4, - n_out=500, activation=T.tanh) + layer2 = HiddenLayer( + rng, + input = layer2_input, + n_in = nkerns[1] * 4 * 4, + n_out = 500, + activation = T.tanh + ) # classify the values of the fully-connected sigmoidal layer - layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10) + layer3 = LogisticRegression(input = layer2.output, n_in = 500, n_out = 10) # the cost we minimize during training is the NLL of the model cost = layer3.negative_log_likelihood(y) # create a function to compute the mistakes that are made by the model - test_model = theano.function([index], layer3.errors(y), - givens={ - x: test_set_x[index * batch_size: (index + 1) * batch_size], - y: test_set_y[index * batch_size: (index + 1) * batch_size]}) - - validate_model = theano.function([index], layer3.errors(y), - givens={ - x: valid_set_x[index * batch_size: (index + 1) * batch_size], - y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) + test_model = theano.function( + [index], + layer3.errors(y), + givens = { + x: test_set_x[index * batch_size : (index + 1) * batch_size], + y: test_set_y[index * batch_size : (index + 1) * batch_size] + } + ) + + validate_model = theano.function( + [index], + layer3.errors(y), + givens = { + x: valid_set_x[index * batch_size : (index + 1) * batch_size], + y: valid_set_y[index * batch_size : (index + 1) * batch_size] + } + ) # create a list of all model parameters to be fit by gradient descent params = layer3.params + layer2.params + layer1.params + layer0.params @@ -208,14 +239,20 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # manually create an update rule for each model parameter. We thus # create the updates list by automatically looping over all # (params[i],grads[i]) pairs. - updates = [] - for param_i, grad_i in zip(params, grads): - updates.append((param_i, param_i - learning_rate * grad_i)) - - train_model = theano.function([index], cost, updates=updates, - givens={ - x: train_set_x[index * batch_size: (index + 1) * batch_size], - y: train_set_y[index * batch_size: (index + 1) * batch_size]}) + updates = [ + (param_i, param_i - learning_rate * grad_i) + for param_i, grad_i in zip(params, grads) + ] + + train_model = theano.function( + [index], + cost, + updates = updates, + givens = { + x: train_set_x[index * batch_size : (index + 1) * batch_size], + y: train_set_y[index * batch_size : (index + 1) * batch_size] + } + ) ############### # TRAIN MODEL # diff --git a/code/dA.py b/code/dA.py index 86b7fe86..f3ae58b2 100644 --- a/code/dA.py +++ b/code/dA.py @@ -75,9 +75,17 @@ class dA(object): """ - def __init__(self, numpy_rng, theano_rng=None, input=None, - n_visible=784, n_hidden=500, - W=None, bhid=None, bvis=None): + def __init__( + self, + numpy_rng, + theano_rng = None, + input = None, + n_visible = 784, + n_hidden = 500, + W = None, + bhid = None, + bvis = None + ): """ Initialize the dA class by specifying the number of visible units (the dimension d of the input ), the number of hidden units ( the dimension @@ -232,9 +240,10 @@ def get_cost_updates(self, corruption_level, learning_rate): # to its parameters gparams = T.grad(cost, self.params) # generate the list of updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - learning_rate * gparam)) + updates = [ + (param, param - learning_rate * gparam) + for param, gparam in zip(self.params, gparams) + ] return (cost, updates) @@ -277,15 +286,27 @@ def test_dA(learning_rate=0.1, training_epochs=15, rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) - da = dA(numpy_rng=rng, theano_rng=theano_rng, input=x, - n_visible=28 * 28, n_hidden=500) - - cost, updates = da.get_cost_updates(corruption_level=0., - learning_rate=learning_rate) - - train_da = theano.function([index], cost, updates=updates, - givens={x: train_set_x[index * batch_size: - (index + 1) * batch_size]}) + da = dA( + numpy_rng = rng, + theano_rng = theano_rng, + input = x, + n_visible = 28 * 28, + n_hidden = 500 + ) + + cost, updates = da.get_cost_updates( + corruption_level=0., + learning_rate=learning_rate + ) + + train_da = theano.function( + [index], + cost, + updates = updates, + givens = { + x: train_set_x[index * batch_size : (index + 1) * batch_size] + } + ) start_time = time.clock() @@ -322,11 +343,18 @@ def test_dA(learning_rate=0.1, training_epochs=15, rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) - da = dA(numpy_rng=rng, theano_rng=theano_rng, input=x, - n_visible=28 * 28, n_hidden=500) - - cost, updates = da.get_cost_updates(corruption_level=0.3, - learning_rate=learning_rate) + da = dA( + numpy_rng = rng, + theano_rng = theano_rng, + input = x, + n_visible = 28 * 28, + n_hidden = 500 + ) + + cost, updates = da.get_cost_updates( + corruption_level = 0.3, + learning_rate=learning_rate + ) train_da = theano.function([index], cost, updates=updates, givens={x: train_set_x[index * batch_size: diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 074d4c13..be6c4358 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -74,20 +74,34 @@ def __init__(self, input, n_in, n_out): """ # initialize with 0 the weights W as a matrix of shape (n_in, n_out) - self.W = theano.shared(value=numpy.zeros((n_in, n_out), - dtype=theano.config.floatX), - name='W', borrow=True) + self.W = theano.shared( + value = numpy.zeros( + (n_in, n_out), + dtype = theano.config.floatX + ), + name = 'W', + borrow = True + ) # initialize the baises b as a vector of n_out 0s - self.b = theano.shared(value=numpy.zeros((n_out,), - dtype=theano.config.floatX), - name='b', borrow=True) - - # compute vector of class-membership probabilities in symbolic form + self.b = theano.shared( + value = numpy.zeros( + (n_out,), + dtype = theano.config.floatX + ), + name = 'b', + borrow = True + ) + + # symbolic expression for computing the matrix of class-membership probabilities + # Where: + # W is a matrix where column-k represent the separation hyper plain for class-k + # x is a matrix where row-j represents input training sample-j + # b is a vector where element-k represent the free parameter of hyper plain-k self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b) - # compute prediction as class whose probability is maximal in - # symbolic form - self.y_pred = T.argmax(self.p_y_given_x, axis=1) + # symbolic description of how to compute prediction as class whose probability + # is maximal + self.y_pred = T.argmax(self.p_y_given_x, axis = 1) # parameters of the model self.params = [self.W, self.b] @@ -255,13 +269,15 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch - x = T.matrix('x') # the data is presented as rasterized images - y = T.ivector('y') # the labels are presented as 1D vector of - # [int] labels + + # generate symbolic variables for input (x and y represent a + # minibatch) + x = T.matrix('x') # data, presented as rasterized images + y = T.ivector('y') # labels, presented as 1D vector of [int] labels # construct the logistic regression class # Each MNIST image has size 28*28 - classifier = LogisticRegression(input=x, n_in=28 * 28, n_out=10) + classifier = LogisticRegression(input = x, n_in = 28 * 28, n_out = 10) # the cost we minimize during training is the negative log likelihood of # the model in symbolic format @@ -269,21 +285,27 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # compiling a Theano function that computes the mistakes that are made by # the model on a minibatch - test_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: test_set_x[index * batch_size: (index + 1) * batch_size], - y: test_set_y[index * batch_size: (index + 1) * batch_size]}) - - validate_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: valid_set_x[index * batch_size:(index + 1) * batch_size], - y: valid_set_y[index * batch_size:(index + 1) * batch_size]}) + test_model = theano.function( + inputs = [index], + outputs = classifier.errors(y), + givens = { + x: test_set_x[index * batch_size : (index + 1) * batch_size], + y: test_set_y[index * batch_size : (index + 1) * batch_size] + } + ) + + validate_model = theano.function( + inputs = [index], + outputs = classifier.errors(y), + givens = { + x: valid_set_x[index * batch_size : (index + 1) * batch_size], + y: valid_set_y[index * batch_size : (index + 1) * batch_size] + } + ) # compute the gradient of cost with respect to theta = (W,b) - g_W = T.grad(cost=cost, wrt=classifier.W) - g_b = T.grad(cost=cost, wrt=classifier.b) + g_W = T.grad(cost = cost, wrt = classifier.W) + g_b = T.grad(cost = cost, wrt = classifier.b) # specify how to update the parameters of the model as a list of # (variable, update expression) pairs. @@ -293,12 +315,15 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # compiling a Theano function `train_model` that returns the cost, but in # the same time updates the parameter of the model based on the rules # defined in `updates` - train_model = theano.function(inputs=[index], - outputs=cost, - updates=updates, - givens={ - x: train_set_x[index * batch_size:(index + 1) * batch_size], - y: train_set_y[index * batch_size:(index + 1) * batch_size]}) + train_model = theano.function( + inputs = [index], + outputs = cost, + updates = updates, + givens = { + x: train_set_x[index * batch_size : (index + 1) * batch_size], + y: train_set_y[index * batch_size : (index + 1) * batch_size] + } + ) ############### # TRAIN MODEL # diff --git a/code/mlp.py b/code/mlp.py index 05c227b0..32cad444 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -79,25 +79,31 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, # We have no info for other function, so we use the same as # tanh. if W is None: - W_values = numpy.asarray(rng.uniform( - low=-numpy.sqrt(6. / (n_in + n_out)), - high=numpy.sqrt(6. / (n_in + n_out)), - size=(n_in, n_out)), dtype=theano.config.floatX) + W_values = numpy.asarray( + rng.uniform( + low = -numpy.sqrt(6. / (n_in + n_out)), + high = numpy.sqrt(6. / (n_in + n_out)), + size = (n_in, n_out) + ), + dtype = theano.config.floatX + ) if activation == theano.tensor.nnet.sigmoid: W_values *= 4 - W = theano.shared(value=W_values, name='W', borrow=True) + W = theano.shared(value = W_values, name = 'W', borrow = True) if b is None: - b_values = numpy.zeros((n_out,), dtype=theano.config.floatX) - b = theano.shared(value=b_values, name='b', borrow=True) + b_values = numpy.zeros((n_out,), dtype = theano.config.floatX) + b = theano.shared(value = b_values, name = 'b', borrow = True) self.W = W self.b = b lin_output = T.dot(input, self.W) + self.b - self.output = (lin_output if activation is None - else activation(lin_output)) + self.output = ( + lin_output if activation is None + else activation(lin_output) + ) # parameters of the model self.params = [self.W, self.b] @@ -140,16 +146,21 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): # into a HiddenLayer with a tanh activation function connected to the # LogisticRegression layer; the activation function can be replaced by # sigmoid or any other nonlinear function - self.hiddenLayer = HiddenLayer(rng=rng, input=input, - n_in=n_in, n_out=n_hidden, - activation=T.tanh) + self.hiddenLayer = HiddenLayer( + rng = rng, + input = input, + n_in = n_in, + n_out = n_hidden, + activation = T.tanh + ) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegression( - input=self.hiddenLayer.output, - n_in=n_hidden, - n_out=n_out) + input = self.hiddenLayer.output, + n_in = n_hidden, + n_out = n_out + ) # L1 norm ; one regularization option is to enforce L1 norm to # be small @@ -227,8 +238,13 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, rng = numpy.random.RandomState(1234) # construct the MLP class - classifier = MLP(rng=rng, input=x, n_in=28 * 28, - n_hidden=n_hidden, n_out=10) + classifier = MLP( + rng = rng, + input = x, + n_in = 28 * 28, + n_hidden = n_hidden, + n_out = 10 + ) # the cost we minimize during training is the negative log likelihood of # the model plus the regularization terms (L1 and L2); cost is expressed @@ -253,29 +269,32 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # compute the gradient of cost with respect to theta (sotred in params) # the resulting gradients will be stored in a list gparams - gparams = [] - for param in classifier.params: - gparam = T.grad(cost, param) - gparams.append(gparam) + gparams = [T.grad(cost, param) for param in classifier.params] # specify how to update the parameters of the model as a list of # (variable, update expression) pairs - updates = [] + # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of # same length, zip generates a list C of same size, where each element # is a pair formed from the two lists : # C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)] - for param, gparam in zip(classifier.params, gparams): - updates.append((param, param - learning_rate * gparam)) + updates = [ + (param, param - learning_rate * gparam) + for param, gparam in zip(classifier.params, gparams) + ] # compiling a Theano function `train_model` that returns the cost, but # in the same time updates the parameter of the model based on the rules # defined in `updates` - train_model = theano.function(inputs=[index], outputs=cost, - updates=updates, - givens={ - x: train_set_x[index * batch_size:(index + 1) * batch_size], - y: train_set_y[index * batch_size:(index + 1) * batch_size]}) + train_model = theano.function( + inputs = [index], + outputs = cost, + updates = updates, + givens = { + x: train_set_x[index * batch_size : (index + 1) * batch_size], + y: train_set_y[index * batch_size : (index + 1) * batch_size] + } + ) ############### # TRAIN MODEL # diff --git a/code/rbm.py b/code/rbm.py index a3013b41..c9f779da 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -27,9 +27,17 @@ class RBM(object): """Restricted Boltzmann Machine (RBM) """ - def __init__(self, input=None, n_visible=784, n_hidden=500, \ - W=None, hbias=None, vbias=None, numpy_rng=None, - theano_rng=None): + def __init__( + self, + input = None, + n_visible = 784, + n_hidden = 500, + W = None, + hbias = None, + vbias = None, + numpy_rng = None, + theano_rng = None + ): """ RBM constructor. Defines the parameters of the model along with basic operations for inferring hidden from visible (and vice-versa), @@ -70,25 +78,38 @@ def __init__(self, input=None, n_visible=784, n_hidden=500, \ # 4*sqrt(6./(n_hidden+n_visible)) the output of uniform if # converted using asarray to dtype theano.config.floatX so # that the code is runable on GPU - initial_W = numpy.asarray(numpy_rng.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), - dtype=theano.config.floatX) + initial_W = numpy.asarray( + numpy_rng.uniform( + low = -4 * numpy.sqrt(6. / (n_hidden + n_visible)), + high = 4 * numpy.sqrt(6. / (n_hidden + n_visible)), + size = (n_visible, n_hidden) + ), + dtype = theano.config.floatX + ) # theano shared variables for weights and biases - W = theano.shared(value=initial_W, name='W', borrow=True) + W = theano.shared(value = initial_W, name = 'W', borrow = True) if hbias is None: # create shared variable for hidden units bias - hbias = theano.shared(value=numpy.zeros(n_hidden, - dtype=theano.config.floatX), - name='hbias', borrow=True) + hbias = theano.shared( + value = numpy.zeros( + n_hidden, + dtype = theano.config.floatX + ), + name = 'hbias', + borrow=True + ) if vbias is None: # create shared variable for visible units bias - vbias = theano.shared(value=numpy.zeros(n_visible, - dtype=theano.config.floatX), - name='vbias', borrow=True) + vbias = theano.shared( + value = numpy.zeros( + n_visible, + dtype = theano.config.floatX + ), + name = 'vbias', + borrow = True + ) # initialize input layer for standalone RBM or layer0 of DBN self.input = input diff --git a/code/rnnrbm.py b/code/rnnrbm.py index b55388b8..9cef86c8 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -28,29 +28,29 @@ def build_rbm(v, W, bv, bh, k): '''Construct a k-step Gibbs chain starting at v for an RBM. -v : Theano vector or matrix - If a matrix, multiple chains will be run in parallel (batch). -W : Theano matrix - Weight matrix of the RBM. -bv : Theano vector - Visible bias vector of the RBM. -bh : Theano vector - Hidden bias vector of the RBM. -k : scalar or Theano scalar - Length of the Gibbs chain. - -Return a (v_sample, cost, monitor, updates) tuple: - -v_sample : Theano vector or matrix with the same shape as `v` - Corresponds to the generated sample(s). -cost : Theano scalar - Expression whose gradient with respect to W, bv, bh is the CD-k approximation - to the log-likelihood of `v` (training example) under the RBM. - The cost is averaged in the batch case. -monitor: Theano scalar - Pseudo log-likelihood (also averaged in the batch case). -updates: dictionary of Theano variable -> Theano variable - The `updates` object returned by scan.''' + v : Theano vector or matrix + If a matrix, multiple chains will be run in parallel (batch). + W : Theano matrix + Weight matrix of the RBM. + bv : Theano vector + Visible bias vector of the RBM. + bh : Theano vector + Hidden bias vector of the RBM. + k : scalar or Theano scalar + Length of the Gibbs chain. + + Return a (v_sample, cost, monitor, updates) tuple: + + v_sample : Theano vector or matrix with the same shape as `v` + Corresponds to the generated sample(s). + cost : Theano scalar + Expression whose gradient with respect to W, bv, bh is the CD-k approximation + to the log-likelihood of `v` (training example) under the RBM. + The cost is averaged in the batch case. + monitor: Theano scalar + Pseudo log-likelihood (also averaged in the batch case). + updates: dictionary of Theano variable -> Theano variable + The `updates` object returned by scan.''' def gibbs_step(v): mean_h = T.nnet.sigmoid(T.dot(v, W) + bh) @@ -78,7 +78,7 @@ def free_energy(v): def shared_normal(num_rows, num_cols, scale=1): '''Initialize a matrix shared variable with normally distributed -elements.''' + elements.''' return theano.shared(numpy.random.normal( scale=scale, size=(num_rows, num_cols)).astype(theano.config.floatX)) @@ -91,36 +91,36 @@ def shared_zeros(*shape): def build_rnnrbm(n_visible, n_hidden, n_hidden_recurrent): '''Construct a symbolic RNN-RBM and initialize parameters. -n_visible : integer - Number of visible units. -n_hidden : integer - Number of hidden units of the conditional RBMs. -n_hidden_recurrent : integer - Number of hidden units of the RNN. - -Return a (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) tuple: - -v : Theano matrix - Symbolic variable holding an input sequence (used during training) -v_sample : Theano matrix - Symbolic variable holding the negative particles for CD log-likelihood - gradient estimation (used during training) -cost : Theano scalar - Expression whose gradient (considering v_sample constant) corresponds to the - LL gradient of the RNN-RBM (used during training) -monitor : Theano scalar - Frame-level pseudo-likelihood (useful for monitoring during training) -params : tuple of Theano shared variables - The parameters of the model to be optimized during training. -updates_train : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - training function. -v_t : Theano matrix - Symbolic variable holding a generated sequence (used during sampling) -updates_generate : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - generation function.''' + n_visible : integer + Number of visible units. + n_hidden : integer + Number of hidden units of the conditional RBMs. + n_hidden_recurrent : integer + Number of hidden units of the RNN. + + Return a (v, v_sample, cost, monitor, params, updates_train, v_t, + updates_generate) tuple: + + v : Theano matrix + Symbolic variable holding an input sequence (used during training) + v_sample : Theano matrix + Symbolic variable holding the negative particles for CD log-likelihood + gradient estimation (used during training) + cost : Theano scalar + Expression whose gradient (considering v_sample constant) corresponds to the + LL gradient of the RNN-RBM (used during training) + monitor : Theano scalar + Frame-level pseudo-likelihood (useful for monitoring during training) + params : tuple of Theano shared variables + The parameters of the model to be optimized during training. + updates_train : dictionary of Theano variable -> Theano variable + Update object that should be passed to theano.function when compiling the + training function. + v_t : Theano matrix + Symbolic variable holding a generated sequence (used during sampling) + updates_generate : dictionary of Theano variable -> Theano variable + Update object that should be passed to theano.function when compiling the + generation function.''' W = shared_normal(n_visible, n_hidden, 0.01) bv = shared_zeros(n_visible) @@ -174,27 +174,33 @@ def recurrence(v_t, u_tm1): class RnnRbm: '''Simple class to train an RNN-RBM from MIDI files and to generate sample -sequences.''' - - def __init__(self, n_hidden=150, n_hidden_recurrent=100, lr=0.001, - r=(21, 109), dt=0.3): + sequences.''' + + def __init__( + self, + n_hidden = 150, + n_hidden_recurrent = 100, + lr = 0.001, + r = (21, 109), + dt = 0.3 + ): '''Constructs and compiles Theano functions for training and sequence -generation. - -n_hidden : integer - Number of hidden units of the conditional RBMs. -n_hidden_recurrent : integer - Number of hidden units of the RNN. -lr : float - Learning rate -r : (integer, integer) tuple - Specifies the pitch range of the piano-roll in MIDI note numbers, including - r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the - RBM at a given time step. The default (21, 109) corresponds to the full range - of piano (88 notes). -dt : float - Sampling period when converting the MIDI files into piano-rolls, or - equivalently the time difference between consecutive time steps.''' + generation. + + n_hidden : integer + Number of hidden units of the conditional RBMs. + n_hidden_recurrent : integer + Number of hidden units of the RNN. + lr : float + Learning rate + r : (integer, integer) tuple + Specifies the pitch range of the piano-roll in MIDI note numbers, including + r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the + RBM at a given time step. The default (21, 109) corresponds to the full range + of piano (88 notes). + dt : float + Sampling period when converting the MIDI files into piano-rolls, or + equivalently the time difference between consecutive time steps.''' self.r = r self.dt = dt @@ -212,16 +218,16 @@ def __init__(self, n_hidden=150, n_hidden_recurrent=100, lr=0.001, def train(self, files, batch_size=100, num_epochs=200): '''Train the RNN-RBM via stochastic gradient descent (SGD) using MIDI -files converted to piano-rolls. + files converted to piano-rolls. -files : list of strings - List of MIDI files that will be loaded as piano-rolls for training. -batch_size : integer - Training sequences will be split into subsequences of at most this size - before applying the SGD updates. -num_epochs : integer - Number of epochs (pass over the training set) performed. The user can - safely interrupt training with Ctrl+C at any time.''' + files : list of strings + List of MIDI files that will be loaded as piano-rolls for training. + batch_size : integer + Training sequences will be split into subsequences of at most this size + before applying the SGD updates. + num_epochs : integer + Number of epochs (pass over the training set) performed. The user can + safely interrupt training with Ctrl+C at any time.''' assert len(files) > 0, 'Training set is empty!' \ ' (did you download the data files?)' @@ -248,12 +254,12 @@ def train(self, files, batch_size=100, num_epochs=200): def generate(self, filename, show=True): '''Generate a sample sequence, plot the resulting piano-roll and save -it as a MIDI file. + it as a MIDI file. -filename : string - A MIDI file will be created at this location. -show : boolean - If True, a piano-roll of the generated sequence will be shown.''' + filename : string + A MIDI file will be created at this location. + show : boolean + If True, a piano-roll of the generated sequence will be shown.''' piano_roll = self.generate_function() midiwrite(filename, piano_roll, self.r, self.dt) diff --git a/doc/DBN.txt b/doc/DBN.txt index b2e5230a..017585eb 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -4,7 +4,7 @@ Deep Belief Networks ==================== .. note:: - This section assumes the reader has already read through :doc:`logreg` + This section assumes the reader has already read through :doc:`logistic_sgd` and :doc:`mlp` and :doc:`rbm`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the @@ -142,47 +142,9 @@ the RBMs to initialize an MLP, the code will reflect this by seperating as much as possible the RBMs used to initialize the network and the MLP used for classification. -.. code-block:: python - - class DBN(object): - - def __init__(self, numpy_rng, theano_rng=None, n_ins=784, - hidden_layers_sizes=[500, 500], n_outs=10): - """This class is made to support a variable number of layers. - - :type numpy_rng: numpy.random.RandomState - :param numpy_rng: numpy random number generator used to draw initial - weights - - :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams - :param theano_rng: Theano random generator; if None is given one is - generated based on a seed drawn from `rng` - - :type n_ins: int - :param n_ins: dimension of the input to the DBN - - :type hidden_layers_sizes: list of ints - :param hidden_layers_sizes: intermediate layers size, must contain - at least one value - - :type n_outs: int - :param n_outs: dimension of the output of the network - """ - - self.sigmoid_layers = [] - self.rbm_layers = [] - self.params = [] - self.n_layers = len(hidden_layers_sizes) - - assert self.n_layers > 0 - - if not theano_rng: - theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) - - # allocate symbolic variables for the data - self.x = T.matrix('x') # the data is presented as rasterized images - self.y = T.ivector('y') # the labels are presented as 1D vector of - # [int] labels +.. literalinclude:: ../code/DBN.py + :start-after: class DBN(object): + :end-before: # The DBN is an MLP ``self.sigmoid_layers`` will store the feed-forward graphs which together form the MLP, while ``self.rbm_layers`` will store the RBMs used to pretrain each @@ -196,134 +158,33 @@ is the depth of our model. We link the sigmoid layers such that they form an MLP, and construct each RBM such that they share the weight matrix and the hidden bias with its corresponding sigmoid layer. - -.. code-block:: python - - for i in xrange(self.n_layers): - # construct the sigmoidal layer - - # the size of the input is either the number of hidden units of the - # layer below or the input size if we are on the first layer - if i == 0: - input_size = n_ins - else: - input_size = hidden_layers_sizes[i - 1] - - # the input to this layer is either the activation of the hidden - # layer below or the input of the DBN if you are on the first layer - if i == 0: - layer_input = self.x - else: - layer_input = self.sigmoid_layers[-1].output - - sigmoid_layer = HiddenLayer(rng=numpy_rng, - input=layer_input, - n_in=input_size, - n_out=hidden_layers_sizes[i], - activation=T.nnet.sigmoid) - - # add the layer to our list of layers - self.sigmoid_layers.append(sigmoid_layer) - - # its arguably a philosophical question... but we are going to only declare that - # the parameters of the sigmoid_layers are parameters of the DBN. The visible - # biases in the RBM are parameters of those RBMs, but not of the DBN. - self.params.extend(sigmoid_layer.params) - - # Construct an RBM that shared weights with this layer - rbm_layer = RBM(numpy_rng=numpy_rng, - theano_rng=theano_rng, - input=layer_input, - n_visible=input_size, - n_hidden=hidden_layers_sizes[i], - W=sigmoid_layer.W, - hbias=sigmoid_layer.b) - self.rbm_layers.append(rbm_layer) - +.. literalinclude:: ../code/DBN.py + :start-after: # MLP. + :end-before: # We now need to add a logistic layer on top of the MLP All that is left is to stack one last logistic regression layer in order to form an MLP. We will use the ``LogisticRegression`` class introduced in -:ref:`logreg`. - -.. code-block:: python - - # We now need to add a logistic layer on top of the MLP - self.logLayer = LogisticRegression( - input=self.sigmoid_layers[-1].output, - n_in=hidden_layers_sizes[-1], n_out=n_outs) - self.params.extend(self.logLayer.params) +:ref:`logistic_sgd`. - # construct a function that implements one step of fine-tuning compute - # the cost for second phase of training, defined as the negative log - # likelihood of the logistic regression (output) layer - self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) - - # compute the gradients with respect to the model parameters - # symbolic variable that points to the number of errors made on the - # minibatch given by self.x and self.y - self.errors = self.logLayer.errors(self.y) +.. literalinclude:: ../code/DBN.py + :start-after: # We now need to add a logistic layer on top of the MLP + :end-before: def pretraining_functions The class also provides a method which generates training functions for each of the RBMs. They are returned as a list, where element :math:`i` is a function which implements one step of training for the ``RBM`` at layer :math:`i`. - -.. code-block:: python - - def pretraining_functions(self, train_set_x, batch_size, k): - ''' Generates a list of functions, for performing one step of gradient descent at a - given layer. The function will require as input the minibatch index, and to train an - RBM you just need to iterate, calling the corresponding function on all minibatch - indexes. - - :type train_set_x: theano.tensor.TensorType - :param train_set_x: Shared var. that contains all datapoints used for training the RBM - :type batch_size: int - :param batch_size: size of a [mini]batch - :param k: number of Gibbs steps to do in CD-k / PCD-k - ''' - - # index to a [mini]batch - index = T.lscalar('index') # index to a minibatch +.. literalinclude:: ../code/DBN.py + :start-after: self.errors = self.logLayer.errors(self.y) + :end-before: learning_rate = T.scalar('lr') In order to be able to change the learning rate during training, we associate a Theano variable to it that has a default value. -.. code-block:: python - - learning_rate = T.scalar('lr') # learning rate to use - - # number of batches - n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size - # begining of a batch, given `index` - batch_begin = index * batch_size - # ending of a batch given `index` - batch_end = batch_begin + batch_size - - pretrain_fns = [] - for rbm in self.rbm_layers: - - # get the cost and the updates list - # using CD-k here (persisent=None) for training each RBM. - # TODO: change cost function to reconstruction error - cost, updates = rbm.cd(learning_rate, persistent=None, k) - - # compile the Theano function; check if k is also a Theano - # variable, if so added to the inputs of the function - if isinstance(k, theano.Variable): - inputs = [index, theano.Param(learning_rate, default=0.1), k] - else: - inputs = index, theano.Param(learning_rate, default=0.1)] - fn = theano.function(inputs=inputs, - outputs=cost, - updates=updates, - givens={self.x: train_set_x[batch_begin: - batch_end]}) - # append `fn` to the list of functions - pretrain_fns.append(fn) - - return pretrain_fns +.. literalinclude:: ../code/DBN.py + :start-after: index = T.lscalar('index') + :end-before: def build_finetune_functions Now any function ``pretrain_fns[i]`` takes as arguments ``index`` and optionally ``lr`` -- the learning rate. Note that the names of the parameters @@ -337,69 +198,8 @@ In the same fashion, the DBN class includes a method for building the functions required for finetuning ( a ``train_model``, a ``validate_model`` and a ``test_model`` function). -.. code-block:: python - - - def build_finetune_functions(self, datasets, batch_size, learning_rate): - '''Generates a function `train` that implements one step of finetuning, a function - `validate` that computes the error on a batch from the validation set, and a function - `test` that computes the error on a batch from the testing set - - :type datasets: list of pairs of theano.tensor.TensorType - :param datasets: It is a list that contain all the datasets; the has to contain three - pairs, `train`, `valid`, `test` in this order, where each pair is formed of two Theano - variables, one for the datapoints, the other for the labels - :type batch_size: int - :param batch_size: size of a minibatch - :type learning_rate: float - :param learning_rate: learning rate used during finetune stage - ''' - - (train_set_x, train_set_y) = datasets[0] - (valid_set_x, valid_set_y) = datasets[1] - (test_set_x, test_set_y) = datasets[2] - - # compute number of minibatches for training, validation and testing - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size - - index = T.lscalar('index') # index to a [mini]batch - - # compute the gradients with respect to the model parameters - gparams = T.grad(self.finetune_cost, self.params) - - # compute list of fine-tuning updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - gparam * learning_rate)) - - train_fn = theano.function(inputs=[index], - outputs= self.finetune_cost, - updates=updates, - givens={ - self.x: train_set_x[index * batch_size: (index + 1) * batch_size], - self.y: train_set_y[index * batch_size: (index + 1) * batch_size]}) - - test_score_i = theano.function([index], self.errors, - givens={ - self.x: test_set_x[index * batch_size: (index + 1) * batch_size], - self.y: test_set_y[index * batch_size: (index + 1) * batch_size]}) - - valid_score_i = theano.function([index], self.errors, - givens={ - self.x: valid_set_x[index * batch_size: (index + 1) * batch_size], - self.y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) - - # Create a function that scans the entire validation set - def valid_score(): - return [valid_score_i(i) for i in xrange(n_valid_batches)] - - # Create a function that scans the entire test set - def test_score(): - return [test_score_i(i) for i in xrange(n_test_batches)] - - return train_fn, valid_score, test_score - +.. literalinclude:: ../code/DBN.py + :pyobject: DBN.build_finetune_functions Note that the returned ``valid_score`` and ``test_score`` are not Theano functions, but rather Python functions. These loop over the entire @@ -412,16 +212,9 @@ Putting it all together The few lines of code below constructs the deep belief network : -.. code-block:: python - - numpy_rng = numpy.random.RandomState(123) - print '... building the model' - # construct the Deep Belief Network - dbn = DBN(numpy_rng=numpy_rng, n_ins=28 * 28, - hidden_layers_sizes=[1000, 1000, 1000], - n_outs=10) - - +.. literalinclude:: ../code/DBN.py + :start-after: # numpy random generator + :end-before: ######################### There are two stages in training this network: (1) a layer-wise pre-training and (2) a fine-tuning stage. @@ -432,34 +225,9 @@ input to the ``i``-th level RBM and performs one step of CD-k within this RBM. This function is applied to the training set for a fixed number of epochs given by ``pretraining_epochs``. - -.. code-block:: python - - - ######################### - # PRETRAINING THE MODEL # - ######################### - print '... getting the pretraining functions' - # We are using CD-1 here - pretraining_fns = dbn.pretraining_functions( - train_set_x=train_set_x, - batch_size=batch_size, - k=k) - - print '... pre-training the model' - start_time = time.clock() - ## Pre-train layer-wise - for i in xrange(dbn.n_layers): - # go through pretraining epochs - for epoch in xrange(pretraining_epochs): - # go through the training set - c = [] - for batch_index in xrange(n_train_batches): - c.append(pretraining_fns[i](index=batch_index, - lr=pretrain_lr)) - print 'Pre-training layer %i, epoch %d, cost '%(i,epoch),numpy.mean(c) - - end_time = time.clock() +.. literalinclude:: ../code/DBN.py + :start-after: PRETRAINING THE MODEL + :end-before: print >> sys.stderr The fine-tuning loop is very similar to the one in the :ref:`mlp` tutorial, the only difference being that we now use the functions given by diff --git a/doc/SdA.txt b/doc/SdA.txt index 02b49d6c..cf012342 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -4,7 +4,7 @@ Stacked Denoising Autoencoders (SdA) ==================================== .. note:: - This section assumes the reader has already read through :doc:`logreg` + This section assumes the reader has already read through :doc:`logistic_sgd` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. @@ -73,54 +73,9 @@ facedes are linked by the fact that the autoencoders and the sigmoid layers of the MLP share parameters, and the fact that autoencoders get as input latent representations of intermediate layers of the MLP. -.. code-block:: python - - class SdA(object): - - def __init__(self, numpy_rng, theano_rng=None, n_ins=784, - hidden_layers_sizes=[500, 500], n_outs=10, - corruption_levels=[0.1, 0.1]): - """ This class is made to support a variable number of layers. - - :type numpy_rng: numpy.random.RandomState - :param numpy_rng: numpy random number generator used to draw initial - weights - - :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams - :param theano_rng: Theano random generator; if None is given one is - generated based on a seed drawn from `rng` - - :type n_ins: int - :param n_ins: dimension of the input to the sdA - - :type n_layers_sizes: list of ints - :param n_layers_sizes: intermediate layers size, must contain - at least one value - - :type n_outs: int - :param n_outs: dimension of the output of the network - - :type corruption_levels: list of float - :param corruption_levels: amount of corruption to use for each - layer - """ - - self.sigmoid_layers = [] - self.dA_layers = [] - self.params = [] - self.n_layers = len(hidden_layers_sizes) - - assert self.n_layers > 0 - - if not theano_rng: - theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) - # allocate symbolic variables for the data - self.x = T.matrix('x') # the data is presented as rasterized images - self.y = T.ivector('y') # the labels are presented as 1D vector of - # [int] labels - - - +.. literalinclude:: ../code/SdA.py + :start-after: class SdA(object): + :end-before: # The SdA is an MLP ``self.sigmoid_layers`` will store the sigmoid layers of the MLP facade, while ``self.dA_layers`` will store the denoising autoencoder associated with the layers of the MLP. @@ -134,74 +89,17 @@ We link the sigmoid layers such that they form an MLP, and construct each denoising autoencoder such that they share the weight matrix and the bias of the encoding part with its corresponding sigmoid layer. -.. code-block:: python - - for i in xrange(self.n_layers): - # construct the sigmoidal layer - - # the size of the input is either the number of hidden units of - # the layer below or the input size if we are on the first layer - if i == 0: - input_size = n_ins - else: - input_size = hidden_layers_sizes[i - 1] - - # the input to this layer is either the activation of the hidden - # layer below or the input of the SdA if you are on the first - # layer - if i == 0: - layer_input = self.x - else: - layer_input = self.sigmoid_layers[-1].output - - sigmoid_layer = HiddenLayer(rng=rng, - input=layer_input, - n_in=input_size, - n_out=hidden_layers_sizes[i], - activation=T.nnet.sigmoid) - # add the layer to our list of layers - self.sigmoid_layers.append(sigmoid_layer) - - # its arguably a philosophical question... - # but we are going to only declare that the parameters of the - # sigmoid_layers are parameters of the StackedDAA - # the visible biases in the dA are parameters of those - # dA, but not the SdA - self.params.extend(sigmoid_layer.params) - - # Construct a denoising autoencoder that shared weights with this - # layer - dA_layer = dA(rng=rng, trng=trng, input=layer_input, - n_visible=input_size, - n_hidden=hidden_layers_sizes[i], - corruption_level=corruption_levels[0], - W=sigmoid_layer.W, - bhid=sigmoid_layer.b) - self.dA_layers.append(dA_layer) - +.. literalinclude:: ../code/SdA.py + :start-after: [int] labels + :end-before: # We now need to add a logistic layer on top of the MLP All we need now is to add the logistic layer on top of the sigmoid layers such that we have an MLP. We will -use the ``LogisticRegression`` class introduced in :ref:`logreg`. - -.. code-block:: python - - # We now need to add a logistic layer on top of the MLP - self.logLayer = LogisticRegression( - input=self.sigmoid_layers[-1].output, - n_in=hidden_layers_sizes[-1], n_out=n_outs) - - self.params.extend(self.logLayer.params) - # construct a function that implements one step of finetunining - - # compute the cost for second phase of training, - # defined as the negative log likelihood - self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) - # compute the gradients with respect to the model parameters - # symbolic variable that points to the number of errors made on the - # minibatch given by self.x and self.y - self.errors = self.logLayer.errors(self.y) +use the ``LogisticRegression`` class introduced in :ref:`logistic_sgd`. +.. literalinclude:: ../code/SdA.py + :start-after: self.dA_layers.append(dA_layer) + :end-before: def pretraining_functions The class also provides a method that generates training functions for each of the denoising autoencoder associated with the different layers. @@ -209,59 +107,16 @@ They are returned as a list, where element :math:`i` is a function that implements one step of training the ``dA`` correspoinding to layer :math:`i`. -.. code-block:: python - - def pretraining_functions(self, train_set_x, batch_size): - ''' Generates a list of functions, each of them implementing one - step in trainnig the dA corresponding to the layer with same index. - The function will require as input the minibatch index, and to train - a dA you just need to iterate, calling the corresponding function on - all minibatch indexes. - - :type train_set_x: theano.tensor.TensorType - :param train_set_x: Shared variable that contains all datapoints used - for training the dA - - :type batch_size: int - :param batch_size: size of a [mini]batch - - :type learning_rate: float - :param learning_rate: learning rate used during training for any of - the dA layers - ''' - - # index to a [mini]batch - index = T.lscalar('index') # index to a minibatch +.. literalinclude:: ../code/SdA.py + :start-after: self.errors = self.logLayer.errors(self.y) + :end-before: corruption_level = T.scalar('corruption') In order to be able to change the corruption level or the learning rate during training we associate a Theano variable to them. -.. code-block:: python - - corruption_level = T.scalar('corruption') # amount of corruption to use - learning_rate = T.scalar('lr') # learning rate to use - # number of batches - n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size - # begining of a batch, given `index` - batch_begin = index * batch_size - # ending of a batch given `index` - batch_end = batch_begin + batch_size - - pretrain_fns = [] - for dA in self.dA_layers: - # get the cost and the updates list - cost, updates = dA.get_cost_updates(corruption_level, learning_rate) - # compile the theano function - fn = theano.function(inputs=[index, - theano.Param(corruption_level, default=0.2), - theano.Param(learning_rate, default=0.1)], - outputs=cost, - updates=updates, - givens={self.x: train_set_x[batch_begin:batch_end]}) - # append `fn` to the list of functions - pretrain_fns.append(fn) - - return pretrain_fns +.. literalinclude:: ../code/SdA.py + :start-after: index = T.lscalar('index') + :end-before: def build_finetune_functions Now any function ``pretrain_fns[i]`` takes as arguments ``index`` and optionally ``corruption`` -- the corruption level or ``lr`` -- the @@ -274,99 +129,23 @@ In the same fashion we build a method for constructing function required during finetuning ( a ``train_model``, a ``validate_model`` and a ``test_model`` function). -.. code-block:: python - - def build_finetune_functions(self, datasets, batch_size, learning_rate): - '''Generates a function `train` that implements one step of - finetuning, a function `validate` that computes the error on - a batch from the validation set, and a function `test` that - computes the error on a batch from the testing set - - :type datasets: list of pairs of theano.tensor.TensorType - :param datasets: It is a list that contain all the datasets; - the has to contain three pairs, `train`, - `valid`, `test` in this order, where each pair - is formed of two Theano variables, one for the - datapoints, the other for the labels - - :type batch_size: int - :param batch_size: size of a minibatch - - :type learning_rate: float - :param learning_rate: learning rate used during finetune stage - ''' - - (train_set_x, train_set_y) = datasets[0] - (valid_set_x, valid_set_y) = datasets[1] - (test_set_x, test_set_y) = datasets[2] - - # compute number of minibatches for training, validation and testing - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size - - index = T.lscalar('index') # index to a [mini]batch - - # compute the gradients with respect to the model parameters - gparams = T.grad(self.finetune_cost, self.params) - - # compute list of fine-tuning updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - gparam * learning_rate)) - - train_fn = theano.function(inputs=[index], - outputs=self.finetune_cost, - updates=updates, - givens={ - self.x: train_set_x[index * batch_size: (index + 1) * batch_size], - self.y: train_set_y[index * batch_size: (index + 1) * batch_size]}) - - test_score_i = theano.function([index], self.errors, - givens={ - self.x: test_set_x[index * batch_size: (index+1) * batch_size], - self.y: test_set_y[index * batch_size: (index+1) * batch_size]}) - - valid_score_i = theano.function([index], self.errors, - givens={ - self.x: valid_set_x[index * batch_size: (index + 1) * batch_size], - self.y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) - - # Create a function that scans the entire validation set - def valid_score(): - return [valid_score_i(i) for i in xrange(n_valid_batches)] - - # Create a function that scans the entire test set - def test_score(): - return [test_score_i(i) for i in xrange(n_test_batches)] - - return train_fn, valid_score, test_score - - +.. literalinclude:: ../code/SdA.py + :pyobject: SdA.build_finetune_functions Note that the returned ``valid_score`` and ``test_score`` are not Theano functions, but rather python functions that also loop over the entire validation set and the entire test set producing a list of the losses over these sets. - - - Putting it all together +++++++++++++++++++++++ The few lines of code below constructs the stacked denoising autoencoder : -.. code-block:: python - - numpy_rng = numpy.random.RandomState(123) - print '... building the model' - # construct the stacked denoising autoencoder class - sda = SdA(numpy_rng=numpy_rng, n_ins=28 * 28, - hidden_layers_sizes=[100, 100, 100], - n_outs=10) - - +.. literalinclude:: ../code/SdA.py + :start-after: # numpy random generator + :end-before: PRETRAINING THE MODEL There are two stages in training this network, a layer-wise pre-training and fine-tuning afterwards. @@ -378,44 +157,14 @@ the reconstruction cost of that layer. This function will be applied to the training set for a fixed number of epochs given by ``pretraining_epochs``. - -.. code-block:: python - - ######################### - # PRETRAINING THE MODEL # - ######################### - print '... getting the pretraining functions' - pretraining_fns = sda.pretraining_functions( - train_set_x=train_set_x, - batch_size=batch_size) - - print '... pre-training the model' - start_time = time.clock() - ## Pre-train layer-wise - for i in xrange(sda.n_layers): - # go through pretraining epochs - for epoch in xrange(pretraining_epochs): - # go through the training set - c = [] - for batch_index in xrange(n_train_batches): - c.append( pretraining_fns[i](index=batch_index, - corruption=0.2, lr=pretrain_lr ) ) - print 'Pre-training layer %i, epoch %d, cost '%(i,epoch), numpy.mean(c) - - end_time = time.clock() - - print ('Pretraining took %f minutes' %((end_time - start_time) / 60.)) - - - +.. literalinclude:: ../code/SdA.py + :start-after: PRETRAINING THE MODEL + :end-before: FINETUNING THE MODEL The fine-tuning loop is very similar with the one in the :ref:`mlp`, the only difference is that we will use now the functions given by ``build_finetune_functions`` . - - - Running the Code ++++++++++++++++ diff --git a/doc/contents.txt b/doc/contents.txt index 381043d9..3bd48785 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -11,9 +11,9 @@ Contents LICENSE intro gettingstarted - logreg + logistic_sgd mlp - lenet + convolutional_mlp dA SdA rbm diff --git a/doc/convolutional_mlp.txt b/doc/convolutional_mlp.txt index 3035115a..df4c7b32 100644 --- a/doc/convolutional_mlp.txt +++ b/doc/convolutional_mlp.txt @@ -4,7 +4,7 @@ Convolutional Neural Networks (LeNet) ===================================== .. note:: - This section assumes the reader has already read through :doc:`logreg` and + This section assumes the reader has already read through :doc:`logistic_sgd` and :doc:`mlp`. Additionally, it uses the following new Theano functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `floatX`_, `downsample`_ , `conv2d`_, `dimshuffle`_. If you intend to run the @@ -400,147 +400,21 @@ We now have all we need to implement a LeNet model in Theano. We start with the LeNetConvPoolLayer class, which implements a {convolution + max-pooling} layer. -.. code-block:: python - - class LeNetConvPoolLayer(object): - - def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): - """ - Allocate a LeNetConvPoolLayer with shared variable internal parameters. - - :type rng: numpy.random.RandomState - :param rng: a random number generator used to initialize weights - - :type input: theano.tensor.dtensor4 - :param input: symbolic image tensor, of shape image_shape - - :type filter_shape: tuple or list of length 4 - :param filter_shape: (number of filters, num input feature maps, - filter height,filter width) - - :type image_shape: tuple or list of length 4 - :param image_shape: (batch size, num input feature maps, - image height, image width) - - :type poolsize: tuple or list of length 2 - :param poolsize: the downsampling (pooling) factor (#rows,#cols) - """ - assert image_shape[1] == filter_shape[1] - self.input = input - - # initialize weight values: the fan-in of each hidden neuron is - # restricted by the size of the receptive fields. - fan_in = numpy.prod(filter_shape[1:]) - W_values = numpy.asarray(rng.uniform( - low=-numpy.sqrt(3./fan_in), - high=numpy.sqrt(3./fan_in), - size=filter_shape), dtype=theano.config.floatX) - self.W = theano.shared(value=W_values, name='W') - - # the bias is a 1D tensor -- one bias per output feature map - b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) - self.b = theano.shared(value=b_values, name='b') - - # convolve input feature maps with filters - conv_out = conv.conv2d(input, self.W, - filter_shape=filter_shape, image_shape=image_shape) - - # downsample each feature map individually, using maxpooling - pooled_out = downsample.max_pool_2d(conv_out, poolsize, ignore_border=True) - - # add the bias term. Since the bias is a vector (1D array), we first - # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will thus - # be broadcasted across mini-batches and feature map width & height - self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) - - # store parameters of this layer - self.params = [self.W, self.b] - +.. literalinclude:: ../code/convolutional_mlp.py + :pyobject: LeNetConvPoolLayer Notice that when initializing the weight values, the fan-in is determined by the size of the receptive fields and the number of input feature maps. -Finally, using the LogisticRegression class defined in :doc:`logreg` and +Finally, using the LogisticRegression class defined in :doc:`logistic_sgd` and the HiddenLayer class defined in :doc:`mlp` , we can instantiate the network as follows. -.. code-block:: python - - learning_rate = 0.1 - rng = numpy.random.RandomState(23455) - - ishape = (28, 28) # this is the size of MNIST images - batch_size = 20 # sized of the minibatch - - # allocate symbolic variables for the data - x = T.matrix('x') # rasterized images - y = T.lvector('y') # the labels are presented as 1D vector of [long int] labels - - ############################## - # BEGIN BUILDING ACTUAL MODE - ############################## - - # Reshape matrix of rasterized images of shape (batch_size,28*28) - # to a 4D tensor, compatible with our LeNetConvPoolLayer - layer0_input = x.reshape((batch_size,1,28,28)) - - # Construct the first convolutional pooling layer: - # filtering reduces the image size to (28-5+1,28-5+1)=(24,24) - # maxpooling reduces this further to (24/2,24/2) = (12,12) - # 4D output tensor is thus of shape (20,20,12,12) - layer0 = LeNetConvPoolLayer(rng, input=layer0_input, - image_shape=(batch_size, 1, 28, 28), - filter_shape=(20, 1, 5, 5), poolsize=(2, 2)) - - # Construct the second convolutional pooling layer - # filtering reduces the image size to (12 - 5 + 1, 12 - 5 + 1)=(8, 8) - # maxpooling reduces this further to (8/2,8/2) = (4, 4) - # 4D output tensor is thus of shape (20,50,4,4) - layer1 = LeNetConvPoolLayer(rng, input=layer0.output, - image_shape=(batch_size, 20, 12, 12), - filter_shape=(50, 20, 5, 5), poolsize=(2, 2)) - - # the HiddenLayer being fully-connected, it operates on 2D matrices of - # shape (batch_size,num_pixels) (i.e matrix of rasterized images). - # This will generate a matrix of shape (20, 32 * 4 * 4) = (20, 512) - layer2_input = layer1.output.flatten(2) - - # construct a fully-connected sigmoidal layer - layer2 = HiddenLayer(rng, input=layer2_input, - n_in=50 * 4 * 4, n_out=500, - activation=T.tanh ) - - # classify the values of the fully-connected sigmoidal layer - layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10) - - - # the cost we minimize during training is the NLL of the model - cost = layer3.negative_log_likelihood(y) - - # create a function to compute the mistakes that are made by the model - test_model = theano.function([x, y], layer3.errors(y)) - - # create a list of all model parameters to be fit by gradient descent - params = layer3.params + layer2.params + layer1.params + layer0.params - - # create a list of gradients for all model parameters - grads = T.grad(cost, params) - - # train_model is a function that updates the model parameters by SGD - # Since this model has many parameters, it would be tedious to manually - # create an update rule for each model parameter. We thus create the updates - # dictionary by automatically looping over all (params[i],grads[i]) pairs. - updates = [] - for param_i, grad_i in zip(params, grads): - updates.append((param_i, param_i - learning_rate * grad_i)) - train_model = theano.function([index], cost, updates = updates, - givens={ - x: train_set_x[index * batch_size: (index + 1) * batch_size], - y: train_set_y[index * batch_size: (index + 1) * batch_size]}) - - - -We leave out the code, which performs the actual training and early-stopping, +.. literalinclude:: ../code/convolutional_mlp.py + :start-after: allocate symbolic variables for the data + :end-before: TRAIN MODEL + +We leave out the code that performs the actual training and early-stopping, since it is exactly the same as with an MLP. The interested reader can nevertheless access the code in the 'code' folder of DeepLearningTutorials. diff --git a/doc/dA.txt b/doc/dA.txt index f97140f1..1c59dfa2 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -4,7 +4,7 @@ Denoising Autoencoders (dA) =========================== .. note:: - This section assumes the reader has already read through :doc:`logreg` + This section assumes the reader has already read through :doc:`logistic_sgd` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. @@ -105,89 +105,8 @@ first step is to create shared variables for the parameters of the autoencoder ( :math:`\mathbf{W}`, :math:`\mathbf{b}` and :math:`\mathbf{b'}`, since we are using tied weights in this tutorial ): - - -.. code-block:: python - - class AutoEncoder(object): - - def __init__(self, numpy_rng, input=None, n_visible=784, n_hidden=500, - W=None, bhid=None, bvis=None): - """ - - :type numpy_rng: numpy.random.RandomState - :param numpy_rng: number random generator used to generate weights - - - :type input: theano.tensor.TensorType - :paran input: a symbolic description of the input or None for standalone - dA - - :type n_visible: int - :param n_visible: number of visible units - - :type n_hidden: int - :param n_hidden: number of hidden units - - :type W: theano.tensor.TensorType - :param W: Theano variable pointing to a set of weights that should be - shared belong the dA and another architecture; if dA should - be standalone set this to None - - :type bhid: theano.tensor.TensorType - :param bhid: Theano variable pointing to a set of biases values (for - hidden units) that should be shared belong dA and another - architecture; if dA should be standalone set this to None - - :type bvis: theano.tensor.TensorType - :param bvis: Theano variable pointing to a set of biases values (for - visible units) that should be shared belong dA and another - architecture; if dA should be standalone set this to None - - - """ - self.n_visible = n_visible - self.n_hidden = n_hidden - - - # note : W' was written as `W_prime` and b' as `b_prime` - if not W: - # W is initialized with `initial_W` which is uniformely sampled - # from -4*sqrt(6./(n_visible+n_hidden)) and 4*sqrt(6./(n_hidden+n_visible)) - # the output of uniform if converted using asarray to dtype - # theano.config.floatX so that the code is runable on GPU - initial_W = numpy.asarray(numpy_rng.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), dtype=theano.config.floatX) - W = theano.shared(value=initial_W, name='W') - - if not bvis: - bvis = theano.shared(value=numpy.zeros(n_visible, - dtype=theano.config.floatX), name='bvis') - - if not bhid: - bhid = theano.shared(value=numpy.zeros(n_hidden, - dtype=theano.config.floatX), name='bhid') - - - self.W = W - # b corresponds to the bias of the hidden - self.b = bhid - # b_prime corresponds to the bias of the visible - self.b_prime = bvis - # tied weights, therefore W_prime is W transpose - self.W_prime = self.W.T - # if no input is given, generate a variable representing the input - if input == None: - # we use a matrix because we expect a minibatch of several examples, - # each example being a row - self.x = T.dmatrix(name='input') - else: - self.x = input - - self.params = [self.W, self.b, self.b_prime] - +.. literalinclude:: ../code/dA.py + :pyobject: dA.__init__ Note that we pass the symbolic ``input`` to the autoencoder as a parameter. This is such that later we can concatenate layers of @@ -197,56 +116,25 @@ the k-th layer will be the symbolic input of the (k+1)-th. Now we can express the computation of the latent representation and of the reconstructed signal: -.. code-block:: python +.. literalinclude:: ../code/dA.py + :pyobject: dA.get_hidden_values - def get_hidden_values(self, input): - """ Computes the values of the hidden layer """ - return T.nnet.sigmoid(T.dot(input, self.W) + self.b) - - def get_reconstructed_input(self, hidden): - """ Computes the reconstructed input given the values of the hidden layer """ - return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) +.. literalinclude:: ../code/dA.py + :pyobject: dA.get_reconstructed_input And using these function we can compute the cost and the updates of one stochastic gradient descent step : -.. code-block:: python - - def get_cost_updates(self, learning_rate): - """ This function computes the cost and the updates for one trainng - step """ - - y = self.get_hidden_values(self.x) - z = self.get_reconstructed_input(y) - # note : we sum over the size of a datapoint; if we are using minibatches, - # L will be a vector, with one entry per example in minibatch - L = -T.sum(self.x * T.log(z) + (1 - self.x) * T.log(1 - z), axis=1) - # note : L is now a vector, where each element is the cross-entropy cost - # of the reconstruction of the corresponding example of the - # minibatch. We need to compute the average of all these to get - # the cost of the minibatch - cost = T.mean(L) - - # compute the gradients of the cost of the `dA` with respect - # to its parameters - gparams = T.grad(cost, self.params) - # generate the list of updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - learning_rate * gparam)) - - return (cost, updates) - +.. literalinclude:: ../code/dA.py + :pyobject: dA.get_cost_updates We can now define a function that applied iteratively will update the parameters ``W``, ``b`` and ``b_prime`` such that the reconstruction cost is approximately minimized. -.. code-block:: python - - autoencoder = AutoEncoder(numpy_rng=numpy.random.RandomState(1234), n_visible=784, n_hidden=500) - cost, updates = autoencoder.get_cost_updates(learning_rate=0.1) - train = theano.function([x], cost, updates=updates) +.. literalinclude:: ../code/dA.py + :start-after: theano_rng = RandomStreams(rng.randint(2 ** 30)) + :end-before: start_time = time.clock() One serious potential issue with auto-encoders is that if there is no other constraint besides minimizing the reconstruction error, diff --git a/doc/hmc.txt b/doc/hmc.txt index 2f845509..989c823f 100644 --- a/doc/hmc.txt +++ b/doc/hmc.txt @@ -170,105 +170,16 @@ The inner-loop defined above is implemented by the following `leapfrog` function, with `pos`, `vel` and `step` replacing :math:`s,\phi` and :math:`\epsilon` respectively. -.. code-block:: python - - def leapfrog(pos, vel, step): - """ - Inside loop of Scan. Performs one step of leapfrog update, using - Hamiltonian dynamics. - - Parameters - ---------- - pos: theano matrix - in leapfrog update equations, represents pos(t), position at time t - vel: theano matrix - in leapfrog update equations, represents vel(t - stepsize/2), - velocity at time (t - stepsize/2) - step: theano scalar - scalar value controlling amount by which to move - - Returns - ------- - rval1: [theano matrix, theano matrix] - Symbolic theano matrices for new position pos(t + stepsize), and - velocity vel(t + stepsize/2) - rval2: List of (variable, update expr) pairs - List of updates for the Scan Op - """ - # from pos(t) and vel(t - eps/2), compute vel(t + eps / 2) - dE_dpos = TT.grad(energy_fn(pos).sum(), pos) - new_vel = vel - step * dE_dpos - # from vel(t + eps / 2) compute pos(t + eps) - new_pos = pos + step * new_vel - - return [new_pos, new_vel],{} +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: simulate_dynamics.leapfrog The `simulate_dynamics` function performs the full algorithm of Eqs. :eq:`leap-frog2`. We start with the initial half-step update of :math:`\phi` and full-step of :math:`s`, and then scan over the `leapfrog` method `n\_steps-1` times. -.. code-block:: python - - def simulate_dynamics(initial_pos, initial_vel, stepsize, n_steps, energy_fn): - """ - Return final (position, velocity) obtained after an `n_steps` leapfrog - updates, using Hamiltonian dynamics. - - Parameters - ---------- - initial_pos: shared theano matrix - Initial position at which to start the simulation - initial_vel: shared theano matrix - Initial velocity of particles - stepsize: shared theano scalar - Scalar value controlling amount by which to move - energy_fn: python function - Python function, operating on symbolic theano variables, used to compute - the potential energy at a given position. - - Returns - ------- - rval1: theano matrix - Final positions obtained after simulation - rval2: theano matrix - Final velocity obtained after simulation - """ - - def leapfrog(pos, vel, step): - """ ... """ - - # compute velocity at time-step: t + stepsize / 2 - initial_energy = energy_fn(initial_pos) - dE_dpos = TT.grad(initial_energy.sum(), initial_pos) - vel_half_step = initial_vel - 0.5 * stepsize * dE_dpos - - # compute position at time-step: t + stepsize - pos_full_step = initial_pos + stepsize * vel_half_step - - # perform leapfrog updates: the scan op is used to repeatedly compute - # vel(t + (m-1/2)*stepsize) and pos(t + m*stepsize) for m in [2,n_steps]. - (final_pos, final_vel), scan_updates = theano.scan(leapfrog, - outputs_info=[ - dict(initial=pos_full_step, return_steps=1), - dict(initial=vel_half_step, return_steps=1), - ], - non_sequences=[stepsize], - n_steps=n_steps-1) - - # NOTE: Scan always returns an updates dictionary, in case the scanned function draws - # samples from a RandomStream. These updates must then be used when compiling the Theano - # function, to avoid drawing the same random numbers each time the function is called. In - # this case however, we consciously ignore "scan_updates" because we know it is empty. - assert not scan_updates - - # The last velocity returned by scan is vel(t + (n_steps-1/2)*stepsize) - # We therefore perform one more half-step to return vel(t + n_steps*stepsize) - energy = energy_fn(final_pos) - final_vel = final_vel - 0.5 * stepsize * TT.grad(energy.sum(), final_pos) - - # return new proposal state - return final_pos, final_vel +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: simulate_dynamics A final half-step is performed to compute :math:`\phi(t+n\epsilon)`, and the final proposed state :math:`\chi'` is returned. @@ -283,111 +194,29 @@ energy function :math:`E(s)` (`energy\_fn`), it defines the symbolic graph for computing `n\_steps` of HMC, using a given `stepsize`. The function prototype is as follows: -.. code-block:: python - - def hmc_move(s_rng, positions, energy_fn, stepsize, n_steps): - """ - This function performs one-step of Hybrid Monte-Carlo sampling. We start by - sampling a random velocity from a univariate Gaussian distribution, perform - `n_steps` leap-frog updates using Hamiltonian dynamics and accept-reject - using Metropolis-Hastings. - - Parameters - ---------- - s_rng: theano shared random stream - Symbolic random number generator used to draw random velocity and - perform accept-reject move. - positions: shared theano matrix - Symbolic matrix whose rows are position vectors. - energy_fn: python function - Python function, operating on symbolic theano variables, used to compute - the potential energy at a given position. - stepsize: shared theano scalar - Shared variable containing the stepsize to use for `n_steps` of HMC - simulation steps. - n_steps: integer - Number of HMC steps to perform before proposing a new position. - - Returns - ------- - rval1: boolean - True if move is accepted, False otherwise - rval2: theano matrix - Matrix whose rows contain the proposed "new position" - """ +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: hmc_move We start by sampling random velocities, using the provided shared RandomStream object. Velocities are sampled independently for each dimension and for each particle under simulation, yielding a :math:`N \times D` matrix. -.. code-block:: python - - # sample random velocity for `batchsize` particles - initial_vel = s_rng.normal(size=positions.shape) - - Since we now have an initial position and velocity, we can now call the `simulate\_dynamics` to obtain the proposal for the new state :math:`\chi'`. - -.. code-block:: python - - # perform simulation of particles subject to Hamiltonian dynamics - final_pos, final_vel = simulate_dynamics( - initial_pos = positions, - initial_vel = initial_vel, - stepsize = stepsize, - n_steps = n_steps, - energy_fn = energy_fn) - We then accept/reject the proposed state based on the Metropolis algorithm. -.. code-block:: python - - # accept/reject the proposed move based on the joint distribution - accept = metropolis_hastings_accept( - energy_prev=hamiltonian(positions, initial_vel, energy_fn), - energy_next=hamiltonian(final_pos, final_vel, energy_fn), - s_rng=s_rng) - where `metropolis\_hastings\_accept` and `hamiltonian` are helper functions, defined as follows. -.. code-block:: python - - def metropolis_hastings_accept(energy_prev, energy_next, s_rng): - """ - Performs a Metropolis-Hastings accept-reject move. - - Parameters - ---------- - energy_prev: theano vector - Symbolic theano tensor which contains the energy associated with the - configuration at time-step t. - energy_next: theano vector - Symbolic theano tensor which contains the energy associated with the - proposed configuration at time-step t+1. - s_rng: theano.tensor.shared_randomstreams.RandomStreams - Theano shared random stream object used to generate the random number - used in proposal. - - Returns - ------- - return: boolean - True if move is accepted, False otherwise - """ - ediff = energy_prev - energy_next - return (TT.exp(ediff) - s_rng.uniform(size=energy_prev.shape)) >= 0 - - - def hamiltonian(pos, vel, energy_fn): - """ ... """ - # assuming mass is 1 - return energy_fn(pos) + kinetic_energy(vel) - - def kinetic_energy(vel): - """ ... """ - return 0.5 * (vel ** 2).sum(axis=1) +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: metropolis_hastings_accept + +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: hamiltonian + +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: kinetic_energy `hmc\_move` finally returns the tuple `(accept, final\_pos)`. `accept` is a symbolic boolean variable indicating whether or not the new state `final_pos` @@ -407,17 +236,8 @@ receives as parameters, a series of shared variables to update (`positions`, `st `avg\_acceptance\_rate`), and the parameters required to compute their new state. -.. code-block:: python - - def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, - target_acceptance_rate, stepsize_inc, stepsize_dec, - stepsize_min, stepsize_max, avg_acceptance_slowness): - - ## POSITION UPDATES ## - # broadcast `accept` scalar to tensor with the same dimensions as final_pos. - accept_matrix = accept.dimshuffle(0, *(('x',) * (final_pos.ndim - 1))) - # if accept is True, update to `final_pos` else stay put - new_positions = TT.switch(accept_matrix, final_pos, positions) +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: hmc_updates Using the above code, the dictionary `{positions: new\_positions}` can be used to update the state of the sampler with either (1) the new state `final\_pos` @@ -435,14 +255,6 @@ average acceptance rate of the HMC move proposals (across many simulations), using an exponential moving average with time constant `1-avg\_acceptance\_slowness`. -.. code-block:: python - - ## ACCEPT RATE UPDATES ## - # perform exponential moving average - new_acceptance_rate = TT.add( - avg_acceptance_slowness * avg_acceptance_rate, - (1.0 - avg_acceptance_slowness) * accept.mean()) - If the average acceptance rate is larger than the `target\_acceptance\_rate`, we increase the `stepsize` by a factor of `stepsize\_inc` in order to increase the mixing rate of our chain. If the average acceptance rate is too low however, @@ -450,24 +262,7 @@ mixing rate of our chain. If the average acceptance rate is too low however, conservative mixing rate. The `clip`_ op allows us to maintain the `stepsize` in the range [`stepsize\_min`, `stepsize\_max`]. -.. code-block:: python - - ## STEPSIZE UPDATES ## - # if acceptance rate is too low, our sampler is too "noisy" and we reduce - # the stepsize. If it is too high, our sampler is too conservative, we can - # get away with a larger stepsize (resulting in better mixing). - _new_stepsize = TT.switch(avg_acceptance_rate > target_acceptance_rate, - stepsize * stepsize_inc, stepsize * stepsize_dec) - # maintain stepsize in [stepsize_min, stepsize_max] - new_stepsize = TT.clip(_new_stepsize, stepsize_min, stepsize_max) - -The final updates list is then returned: - -.. code-block:: python - - return [(positions, new_positions), - (stepsize, new_stepsize), - (avg_acceptance_rate, new_acceptance_rate)] +The final updates list is then returned. **HMC_sampler** @@ -481,110 +276,8 @@ elements are: * `draw`: a convenience method which calls the Theano function `simulate` and returns a copy of the contents of the shared variable `self.positions`. - -.. code-block:: python - - class HMC_sampler(object): - """ - Convenience wrapper for performing Hybrid Monte Carlo (HMC). It creates the - symbolic graph for performing an HMC simulation (using `hmc_move` and - `hmc_updates`). The graph is then compiled into the `simulate` function, a - theano function which runs the simulation and updates the required shared - variables. - - Users should interface with the sampler thorugh the `draw` function which - advances the markov chain and returns the current sample by calling - `simulate` and `get_position` in sequence. - - The hyper-parameters are the same as those used by Marc'Aurelio's - 'train_mcRBM.py' file (available on his personal home page). - """ - - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - @classmethod - def new_from_shared_positions(cls, shared_positions, energy_fn, - initial_stepsize=0.01, target_acceptance_rate=.9, n_steps=20, - stepsize_dec=0.98, - stepsize_min=0.001, - stepsize_max=0.25, - stepsize_inc=1.02, - avg_acceptance_slowness=0.9, # used in geometric avg. 1.0 would be not moving at all - seed=12345): - """ - :param shared_positions: theano ndarray shared var with many particle [initial] positions - :param energy_fn: - callable such that energy_fn(positions) - returns theano vector of energies. - The len of this vector is the batchsize. - - The sum of this energy vector must be differentiable (with theano.tensor.grad) with - respect to the positions for HMC sampling to work. - """ - batchsize = shared_positions.shape[0] - - # allocate shared variables - stepsize = sharedX(initial_stepsize, 'hmc_stepsize') - avg_acceptance_rate = sharedX(target_acceptance_rate, 'avg_acceptance_rate') - s_rng = TT.shared_randomstreams.RandomStreams(seed) - - # define graph for an `n_steps` HMC simulation - accept, final_pos = hmc_move( - s_rng, - shared_positions, - energy_fn, - stepsize, - n_steps) - - # define the list of updates, to apply on every `simulate` call - simulate_updates = hmc_updates( - shared_positions, - stepsize, - avg_acceptance_rate, - final_pos=final_pos, - accept=accept, - stepsize_min=stepsize_min, - stepsize_max=stepsize_max, - stepsize_inc=stepsize_inc, - stepsize_dec=stepsize_dec, - target_acceptance_rate=target_acceptance_rate, - avg_acceptance_slowness=avg_acceptance_slowness) - - # compile theano function - simulate = function([], [], updates=simulate_updates) - - # create HMC_sampler object with the following attributes ... - return cls( - positions=shared_positions, - stepsize=stepsize, - stepsize_min=stepsize_min, - stepsize_max=stepsize_max, - avg_acceptance_rate=avg_acceptance_rate, - target_acceptance_rate=target_acceptance_rate, - s_rng=s_rng, - _updates=simulate_updates, - simulate=simulate) - - def draw(self, **kwargs): - """ - Returns a new position obtained after `n_steps` of HMC simulation. - - Parameters - ---------- - kwargs: dictionary - The `kwargs` dictionary is passed to the shared variable - (self.positions) `get_value()` function. For example, to avoid - copying the shared variable value, consider passing `borrow=True`. - - Returns - ------- - rval: numpy matrix - Numpy matrix whose of dimensions similar to `initial_position`. - """ - self.simulate() - return self.positions.get_value(borrow=False) - +.. literalinclude:: ../code/hmc/hmc.py + :pyobject: HMC_sampler Testing our Sampler +++++++++++++++++++ @@ -600,60 +293,11 @@ target energy function. Following a burn-in period, we then generate a large number of samples and compare the empirical mean and covariance matrix to their true values. -.. code-block:: python - - def sampler_on_nd_gaussian(sampler_cls, burnin, n_samples, dim=10): - batchsize=3 - - rng = np.random.RandomState(123) - - # Define a covariance and mu for a gaussian - mu = np.array(rng.rand(dim) * 10, dtype=theano.config.floatX) - cov = np.array(rng.rand(dim, dim), dtype=theano.config.floatX) - cov = (cov + cov.T) / 2. - cov[numpy.arange(dim), numpy.arange(dim)] = 1.0 - cov_inv = linalg.inv(cov) - - # Define energy function for a multi-variate Gaussian - def gaussian_energy(x): - return 0.5 * (TT.dot((x - mu), cov_inv) * (x - mu)).sum(axis=1) - - # Declared shared random variable for positions - position = shared(rng.randn(batchsize, dim).astype(theano.config.floatX)) - - # Create HMC sampler - sampler = sampler_cls(position, gaussian_energy, - initial_stepsize=1e-3, stepsize_max=0.5) - - # Start with a burn-in process - garbage = [sampler.draw() for r in xrange(burnin)] #burn-in - # Draw `n_samples`: result is a 3D tensor of dim [n_samples, batchsize, dim] - _samples = np.asarray([sampler.draw() for r in xrange(n_samples)]) - # Flatten to [n_samples * batchsize, dim] - samples = _samples.T.reshape(dim, -1).T - - print '****** TARGET VALUES ******' - print 'target mean:', mu - print 'target cov:\n', cov - - print '****** EMPIRICAL MEAN/COV USING HMC ******' - print 'empirical mean: ', samples.mean(axis=0) - print 'empirical_cov:\n', np.cov(samples.T) - - print '****** HMC INTERNALS ******' - print 'final stepsize', sampler.stepsize.get_value() - print 'final acceptance_rate', sampler.avg_acceptance_rate.get_value() - - return sampler - - def test_hmc(): - sampler = sampler_on_nd_gaussian(HMC_sampler.new_from_shared_positions, - burnin=1000, n_samples=1000, dim=5) - assert abs(sampler.avg_acceptance_rate.get_value() - - sampler.target_acceptance_rate) < .1 - assert sampler.stepsize.get_value() >= sampler.stepsize_min - assert sampler.stepsize.get_value() <= sampler.stepsize_max +.. literalinclude:: ../code/hmc/test_hmc.py + :pyobject: sampler_on_nd_gaussian +.. literalinclude:: ../code/hmc/test_hmc.py + :pyobject: test_hmc The above code can be run using the command: "nosetests -s code/hmc/test\_hmc.py". The output is as follows: diff --git a/doc/intro.txt b/doc/intro.txt index 19ab4bc7..8573272b 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -29,7 +29,7 @@ read through our :ref:`gettingstarted` chapter -- it introduces the notation, an The purely supervised learning algorithms are meant to be read in order: - #. :ref:`Logistic Regression ` - using Theano for something simple + #. :ref:`Logistic Regression ` - using Theano for something simple #. :ref:`Multilayer perceptron ` - introduction to layers #. :ref:`Deep Convolutional Network ` - a simplified version of LeNet5 diff --git a/doc/logistic_sgd.txt b/doc/logistic_sgd.txt index cd85f252..65956210 100644 --- a/doc/logistic_sgd.txt +++ b/doc/logistic_sgd.txt @@ -1,6 +1,6 @@ .. index:: Logistic Regression -.. _logreg : +.. _logistic_sgd : Classifying MNIST digits using Logistic Regression @@ -53,44 +53,15 @@ The output of the model or prediction is then done by taking the argmax of the v .. math:: y_{pred} = {\rm argmax}_i P(Y=i|x,W,b) - - The code to do this in Theano is the following: -.. code-block:: python - - # generate symbolic variables for input (x and y represent a - # minibatch) - x = T.fmatrix('x') - y = T.lvector('y') - - # allocate shared variables model params - b = theano.shared(numpy.zeros((10,)), name='b') - W = theano.shared(numpy.zeros((784, 10)), name='W') - - # symbolic expression for computing the matrix of class-membership probabilities - # Where: - # W is a matrix where column-k represent the separation hyper plain for class-k - # x is a matrix where row-j represents input training sample-j - # b is a vector where element-k represent the free parameter of hyper plain-k - p_y_given_x = T.nnet.softmax(T.dot(x, W) + b) - - # compiled Theano function that returns the vector of class-membership - # probabilities - get_p_y_given_x = theano.function(inputs=[x], outputs=p_y_given_x) - - # print the probability of some example represented by x_value - # x_value is not a symbolic variable but a numpy array describing the - # datapoint - print 'Probability that x is of class %i is %f' % (i, get_p_y_given_x(x_value)[i]) - - # symbolic description of how to compute prediction as class whose probability - # is maximal - y_pred = T.argmax(p_y_given_x, axis=1) +.. literalinclude:: ../code/logistic_sgd.py + :start-after: index = T.lscalar() + :end-before: # construct the logistic regression class - # compiled theano function that returns this value - classify = theano.function(inputs=[x], outputs=y_pred) +.. literalinclude:: ../code/logistic_sgd.py + :pyobject: LogisticRegression.__init__ We first start by allocating symbolic variables for the inputs :math:`x,y`. Since the parameters of the model must maintain a persistent state throughout @@ -146,13 +117,8 @@ mini-batches (MSGD). See :ref:`opt_SGD` for more details. The following Theano code defines the (symbolic) loss for a given minibatch: -.. code-block:: python - - loss = -T.mean(T.log(p_y_given_x)[T.arange(y.shape[0]), y]) - # note on syntax: T.arange(y.shape[0]) is a vector of integers [0,1,2,...,len(y)]. - # Indexing a matrix M by the two vectors [0,1,...,K], [a,b,...,k] returns the - # elements M[0,a], M[1,b], ..., M[K,k] as a vector. Here, we use this - # syntax to retrieve the log-probability of the correct labels, y. +.. literalinclude:: ../code/logistic_sgd.py + :pyobject: LogisticRegression.negative_log_likelihood .. note:: @@ -169,72 +135,14 @@ We now have all the tools we need to define a ``LogisticRegression`` class, whic encapsulates the basic behaviour of logistic regression. The code is very similar to what we have covered so far, and should be self explanatory. -.. code-block:: python - - class LogisticRegression(object): - - def __init__(self, input, n_in, n_out): - """ Initialize the parameters of the logistic regression - - :type input: theano.tensor.TensorType - :param input: symbolic variable that describes the input of the - architecture (e.g., one minibatch of input images) - - :type n_in: int - :param n_in: number of input units, the dimension of the space in - which the datapoint lies - - :type n_out: int - :param n_out: number of output units, the dimension of the space in - which the target lies - """ - - # initialize with 0 the weights W as a matrix of shape (n_in, n_out) - self.W = theano.shared(value=numpy.zeros((n_in, n_out), - dtype=theano.config.floatX), name='W' ) - # initialize the baises b as a vector of n_out 0s - self.b = theano.shared(value=numpy.zeros((n_out,), - dtype=theano.config.floatX), name='b' ) - - # compute vector of class-membership probabilities in symbolic form - self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b) - - # compute prediction as class whose probability is maximal in - # symbolic form - self.y_pred=T.argmax(self.p_y_given_x, axis=1) - - - def negative_log_likelihood(self, y): - """Return the mean of the negative log-likelihood of the prediction - of this model under a given target distribution. - - .. math:: - - \frac{1}{|\mathcal{D}|} \mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = - \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ - \ell (\theta=\{W,b\}, \mathcal{D}) - - - :param y: corresponds to a vector that gives for each example the - correct label; - - Note: we use the mean instead of the sum so that - the learning rate is less dependent on the batch size - """ - return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y]) - +.. literalinclude:: ../code/logistic_sgd.py + :pyobject: LogisticRegression We instantiate this class as follows: -.. code-block:: python - - # allocate symbolic variables for the data - x = T.fmatrix() # the data is presented as rasterized images (each being a 1-D row vector in x) - y = T.lvector() # the labels are presented as 1D vector of [long int] labels - - # construct the logistic regression class - classifier = LogisticRegression( - input=x.reshape((batch_size, 28 * 28)), n_in=28 * 28, n_out=10) +.. literalinclude:: ../code/logistic_sgd.py + :start-after: index = T.lscalar() + :end-before: # the cost we minimize during Note that the inputs x and y are defined outside the scope of the ``LogisticRegression`` object. Since the class requires the input x to build its @@ -249,9 +157,9 @@ to the other. The last step involves defining a (symbolic) cost variable to minimize, using the instance method ``classifier.negative_log_likelihood``. -.. code-block:: python - - cost = classifier.negative_log_likelihood(y) +.. literalinclude:: ../code/logistic_sgd.py + :start-after: classifier = LogisticRegression(input = x, n_in = 28 * 28, n_out = 10) + :end-before: # compiling a Theano function that computes the mistakes Note how x is an implicit symbolic input to the symbolic definition of cost, here, because classifier.__init__ has defined its symbolic variables in terms of x. @@ -274,38 +182,17 @@ numerical stability. To get the gradients :math:`\partial{\ell}/\partial{W}` and :math:`\partial{\ell}/\partial{b}` in Theano, simply do the following: -.. code-block:: python - - # compute the gradient of cost with respect to theta = (W,b) - g_W = T.grad(cost, classifier.W) - g_b = T.grad(cost, classifier.b) +.. literalinclude:: ../code/logistic_sgd.py + :start-after: # compute the gradient of cost + :end-before: # specify how to update the parameters ``g_W`` and ``g_b`` are again symbolic variables, which can be used as part of a computation graph. Performing one-step of gradient descent can then be done as follows: -.. code-block:: python - - # compute the gradient of cost with respect to theta = (W,b) - g_W = T.grad(cost=cost, wrt=classifier.W) - g_b = T.grad(cost=cost, wrt=classifier.b) - - # specify how to update the parameters of the model as a list of - # (variable, update expression) pairs - updates = [(classifier.W, classifier.W - learning_rate * g_W), - (classifier.b, classifier.b - learning_rate * g_b)] - - # compiling a Theano function `train_model` that returns the cost, but in - # the same time updates the parameter of the model based on the rules - # defined in `updates` - train_model = theano.function(inputs=[index], - outputs=cost, - updates=updates, - givens={ - x: train_set_x[index * batch_size: (index + 1) * batch_size], - y: train_set_y[index * batch_size: (index + 1) * batch_size]}) - - +.. literalinclude:: ../code/logistic_sgd.py + :start-after: g_b = T.grad(cost = cost, wrt = classifier.b) + :end-before: # TRAIN MODEL The ``updates`` list contains, for each parameter, the stochastic gradient update operation. The ``givens`` dictionary indicates with @@ -340,19 +227,8 @@ each minibatch. The code is as follows: -.. code-block:: python - - class LogisticRegression(object): - - ... - - def errors(self, y): - """Return a float representing the number of errors in the minibatch - over the total number of examples of the minibatch ; zero - one loss over the size of the minibatch - """ - return T.mean(T.neq(self.y_pred, y)) - +.. literalinclude:: ../code/logistic_sgd.py + :pyobject: LogisticRegression.errors We then create a function ``test_model`` and a function ``validate_model``, which we can call to retrieve this value. As you will see shortly, ``validate_model`` is key to our early-stopping @@ -362,23 +238,9 @@ missclassified examples for that mini-batch. The only difference between them is that one draws its batches from the testing set, while the other from the validation set. -.. code-block:: python - - # compiling a Theano function that computes the mistakes that are made by - # the model on a minibatch - test_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: test_set_x[index * batch_size: (index + 1) * batch_size], - y: test_set_y[index * batch_size: (index + 1) * batch_size]}) - - validate_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: valid_set_x[index * batch_size: (index + 1) * batch_size], - y: valid_set_y[index * batch_size: (index + 1) * batch_size]}) - - +.. literalinclude:: ../code/logistic_sgd.py + :start-after: cost = classifier.negative_log_likelihood(y) + :end-before: # compute the gradient of cost Putting it All Together +++++++++++++++++++++++ @@ -419,5 +281,3 @@ instance we used a batch size of 600. `logistic_cg.py `_ demonstrates how to use SciPy's conjugate gradient solver with Theano on the logistic regression task. - - diff --git a/doc/mlp.txt b/doc/mlp.txt index 4772d1ee..03884286 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -7,7 +7,7 @@ Multilayer Perceptron ===================== .. note:: - This section assumes the reader has already read through :doc:`logreg`. + This section assumes the reader has already read through :doc:`logistic_sgd`. Additionally, it uses the following new Theano functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, :ref:`L1_L2_regularization`, `floatX`_. If you intend to run the @@ -80,7 +80,7 @@ extension to vectors and tensors consists in applying them element-wise The output vector is then obtained as: :math:`o(x) = G(b^{(2)} + W^{(2)} h(x))`. The reader should recognize the form we already used for -:doc:`logreg`. As before, +:doc:`logistic_sgd`. As before, class-membership probabilities can be obtained by choosing :math:`G` as the :math:`softmax` function (in the case of multi-class classification). @@ -102,40 +102,6 @@ implementing a class that will represent any given hidden layer. To construct the MLP we will then only need to throw a logistic regression layer on top. - -.. code-block:: python - - class HiddenLayer(object): - def __init__(self, rng, input, n_in, n_out, activation=T.tanh): - """ - Typical hidden layer of a MLP: units are fully-connected and have - sigmoidal activation function. Weight matrix W is of shape (n_in,n_out) - and the bias vector b is of shape (n_out,). - - NOTE : The nonlinearity used here is tanh - - Hidden unit activation is given by: tanh(dot(input,W) + b) - - :type rng: numpy.random.RandomState - :param rng: a random number generator used to initialize weights - - :type input: theano.tensor.dmatrix - :param input: a symbolic tensor of shape (n_examples, n_in) - - :type n_in: int - :param n_in: dimensionality of input - - :type n_out: int - :param n_out: number of hidden units - - :type activation: theano.Op or function - :param activation: Non linearity to be applied in the hidden - layer - """ - self.input = input - - - The initial values for the weights of a hidden layer :math:`i` should be uniformly sampled from a symmetric interval that depends on the activation function. For :math:`tanh` activation function results obtained in [Xavier10]_ show that the @@ -149,133 +115,30 @@ regime of its activation function where information can easily be propagated both upward (activations flowing from inputs to outputs) and backward (gradients flowing from outputs to inputs). -.. code-block:: python - - # `W` is initialized with `W_values` which is uniformely sampled - # from sqrt(-6./(n_in+n_hidden)) and sqrt(6./(n_in+n_hidden)) - # for tanh activation function - # the output of uniform is converted using asarray to dtype - # theano.config.floatX so that the code is runable on GPU - # Note : optimal initialization of weights is dependent on the - # activation function used (among other things). - # For example, results presented in [Xavier10]_ suggest that you - # should use 4 times larger initial weights for sigmoid - # compared to tanh - # We have no info for other function, so we use the same as tanh. - W_values = numpy.asarray(rng.uniform( - low=-numpy.sqrt(6. / (n_in + n_out)), - high=numpy.sqrt(6. / (n_in + n_out)), - size=(n_in, n_out)), dtype=theano.config.floatX) - if activation == theano.tensor.nnet.sigmoid: - W_values *= 4 - - self.W = theano.shared(value=W_values, name='W') - - b_values = numpy.zeros((n_out,), dtype=theano.config.floatX) - self.b = theano.shared(value=b_values, name='b') - +.. literalinclude:: ../code/mlp.py + :start-after: self.input = input + :end-before: lin_output = T.dot(input, self.W) + self.b Note that we used a given non-linear function as the activation function of the hidden layer. By default this is ``tanh``, but in many cases we might want to use something else. -.. code-block:: python - - self.output = activation(T.dot(input, self.W) + self.b) - # parameters of the model - self.params = [self.W, self.b] +.. literalinclude:: ../code/mlp.py + :start-after: self.b = b + :end-before: # parameters of the model If you look into theory this class implements the graph that computes the hidden layer value :math:`h(x) = \Phi(x) = s(b^{(1)} + W^{(1)} x)`. If you give this as input to the ``LogisticRegression`` class, -implemented in the previous tutorial :doc:`logreg`, you get the output +implemented in the previous tutorial :doc:`logistic_sgd`, you get the output of the MLP. You can see this in the following short implementation of -the ``MLP`` class : - -.. code-block:: python - - class MLP(object): - """Multi-Layer Perceptron Class - - A multilayer perceptron is a feedforward artificial neural network model - that has one layer or more of hidden units and nonlinear activations. - Intermediate layers usually have as activation function tanh or the - sigmoid function (defined here by a ``HiddenLayer`` class) while the - top layer is a softamx layer (defined here by a ``LogisticRegression`` - class). - """ - - - - def __init__(self, rng, input, n_in, n_hidden, n_out): - """Initialize the parameters for the multilayer perceptron - - :type rng: numpy.random.RandomState - :param rng: a random number generator used to initialize weights - - :type input: theano.tensor.TensorType - :param input: symbolic variable that describes the input of the - architecture (one minibatch) - - :type n_in: int - :param n_in: number of input units, the dimension of the space in - which the datapoints lie - - :type n_hidden: int - :param n_hidden: number of hidden units - - :type n_out: int - :param n_out: number of output units, the dimension of the space in - which the labels lie - - """ - - # Since we are dealing with a one hidden layer MLP, this will - # translate into a Hidden Layer connected to the LogisticRegression - # layer - self.hiddenLayer = HiddenLayer(rng = rng, input = input, - n_in = n_in, n_out = n_hidden, - activation = T.tanh) - - # The logistic regression layer gets as input the hidden units - # of the hidden layer - self.logRegressionLayer = LogisticRegression( - input=self.hiddenLayer.output, - n_in=n_hidden, - n_out=n_out) - +the ``MLP`` class. In this tutorial we will also use L1 and L2 regularization (see :ref:`L1_L2_regularization`). For this, we need to compute the L1 norm and the squared L2 norm of the weights :math:`W^{(1)}, W^{(2)}`. -.. code-block:: python - - # L1 norm ; one regularization option is to enforce L1 norm to - # be small - self.L1 = abs(self.hiddenLayer.W).sum() \ - + abs(self.logRegressionLayer.W).sum() - - # square of L2 norm ; one regularization option is to enforce - # square of L2 norm to be small - self.L2_sqr = (self.hiddenLayer.W ** 2).sum() \ - + (self.logRegressionLayer.W ** 2).sum() - - # negative log likelihood of the MLP is given by the negative - # log likelihood of the output of the model, computed in the - # logistic regression layer - self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood - # same holds for the function computing the number of errors - self.errors = self.logRegressionLayer.errors - - # the parameters of the model are the parameters of the two layer it is - # made out of - self.params = self.hiddenLayer.params + self.logRegressionLayer.params - - - - - - +.. literalinclude:: ../code/mlp.py + :pyobject: MLP As before, we train this model using stochastic gradient descent with mini-batches. The difference is that we modify the cost function to include the @@ -283,15 +146,9 @@ regularization term. ``L1_reg`` and ``L2_reg`` are the hyperparameters controlling the weight of these regularization terms in the total cost function. The code that computes the new cost is: -.. code-block:: python - - # the cost we minimize during training is the negative log likelihood of - # the model plus the regularization terms (L1 and L2); cost is expressed - # here symbolically - cost = classifier.negative_log_likelihood(y) \ - + L1_reg * classifier.L1 \ - + L2_reg * classifier.L2_sqr - +.. literalinclude:: ../code/mlp.py + :start-after: # the cost we minimize during training + :end-before: # compiling a Theano function that computes the mistakes We then update the parameters of the model using the gradient. This code is almost identical to the one for logistic regression. Only the number of @@ -300,37 +157,9 @@ for any number of parameters) we will use the list of parameters that we created with the model ``params`` and parse it, computing a gradient at each step. -.. code-block:: python - - # compute the gradient of cost with respect to theta (stored in params) - # the resulting gradients will be stored in a list gparams - gparams = [] - for param in classifier.params: - gparam = T.grad(cost, param) - gparams.append(gparam) - - - # specify how to update the parameters of the model as a list of - # (variable, update expression) pairs - updates = [] - # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of - # same length, zip generates a list C of same size, where each element - # is a pair formed from the two lists : - # C = [(a1, b1), (a2, b2), (a3, b3) , (a4, b4)] - for param, gparam in zip(classifier.params, gparams): - updates.append((param, param - learning_rate * gparam)) - - - # compiling a Theano function `train_model` that returns the cost, butx - # in the same time updates the parameter of the model based on the rules - # defined in `updates` - train_model = theano.function(inputs=[index], outputs=cost, - updates=updates, - givens={ - x: train_set_x[index * batch_size:(index + 1) * batch_size], - y: train_set_y[index * batch_size:(index + 1) * batch_size]}) - - +.. literalinclude:: ../code/mlp.py + :start-after: # compute the gradient of cost + :end-before: # TRAIN MODEL Putting it All Together +++++++++++++++++++++++ diff --git a/doc/rbm.txt b/doc/rbm.txt index 67f99e5a..a2a962aa 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -5,7 +5,7 @@ Restricted Boltzmann Machines (RBM) .. note:: - This section assumes the reader has already read through :doc:`logreg` + This section assumes the reader has already read through :doc:`logistic_sgd` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_ and `scan`_. If you intend to run the code on GPU also read `GPU`_. @@ -309,136 +309,23 @@ useful when an RBM is used as the building block of a deep network, in which case the weight matrix and the hidden layer bias is shared with the corresponding sigmoidal layer of an MLP network. -.. code-block:: python +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.__init__ - class RBM(object): - """Restricted Boltzmann Machine (RBM) """ - def __init__(self, input=None, n_visible=784, n_hidden=500, - W=None, hbias=None, vbias=None, numpy_rng=None, - theano_rng=None): - """ - RBM constructor. Defines the parameters of the model along with - basic operations for inferring hidden from visible (and vice-versa), - as well as for performing CD updates. - - :param input: None for standalone RBMs or symbolic variable if RBM is - part of a larger graph. - - :param n_visible: number of visible units - - :param n_hidden: number of hidden units - - :param W: None for standalone RBMs or symbolic variable pointing to a - shared weight matrix in case RBM is part of a DBN network; in a DBN, - the weights are shared between RBMs and layers of a MLP - - :param hbias: None for standalone RBMs or symbolic variable pointing - to a shared hidden units bias vector in case RBM is part of a - different network - - :param vbias: None for standalone RBMs or a symbolic variable - pointing to a shared visible units bias - """ - - self.n_visible = n_visible - self.n_hidden = n_hidden - - - if numpy_rng is None: - # create a number generator - numpy_rng = numpy.random.RandomState(1234) - - if theano_rng is None: - theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) - - if W is None : - # W is initialized with `initial_W` which is uniformely sampled - # from -4.*sqrt(6./(n_visible+n_hidden)) and 4.*sqrt(6./(n_hidden+n_visible)) - # the output of uniform if converted using asarray to dtype - # theano.config.floatX so that the code is runable on GPU - initial_W = numpy.asarray(numpy.random.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), - dtype=theano.config.floatX) - # theano shared variables for weights and biases - W = theano.shared(value=initial_W, name='W') - - if hbias is None : - # create shared variable for hidden units bias - hbias = theano.shared(value=numpy.zeros(n_hidden, - dtype=theano.config.floatX), name='hbias') - - if vbias is None : - # create shared variable for visible units bias - vbias = theano.shared(value =numpy.zeros(n_visible, - dtype = theano.config.floatX),name='vbias') - - - # initialize input layer for standalone RBM or layer0 of DBN - self.input = input if input else T.dmatrix('input') +Next step is to define functions which construct the symbolic graph associated +with Eqs. :eq:`rbm_propup` - :eq:`rbm_propdown`. The code is as follows: - self.W = W - self.hbias = hbias - self.vbias = vbias - self.theano_rng = theano_rng - # **** WARNING: It is not a good idea to put things in this list - # other than shared variables created in this function. - self.params = [self.W, self.hbias, self.vbias] +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.propup +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.sample_h_given_v -Next step is to define functions which construct the symbolic graph associated -with Eqs. :eq:`rbm_propup` - :eq:`rbm_propdown`. The code is as follows: +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.propdown -.. code-block:: python - - def propup(self, vis): - ''' This function propagates the visible units activation upwards to - the hidden units - - Note that we return also the pre_sigmoid_activation of the layer. As - it will turn out later, due to how Theano deals with optimization and - stability this symbolic variable will be needed to write down a more - stable graph (see details in the reconstruction cost function) - ''' - pre_sigmoid_activation = T.dot(vis, self.W) + self.hbias - return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)] - - def sample_h_given_v(self, v0_sample): - ''' This function infers state of hidden units given visible units ''' - # compute the activation of the hidden units given a sample of the visibles - pre_sigmoid_h1, h1_mean = self.propup(v0_sample) - # get a sample of the hiddens given their activation - # Note that theano_rng.binomial returns a symbolic sample of dtype - # int64 by default. If we want to keep our computations in floatX - # for the GPU we need to specify to return the dtype floatX - h1_sample = self.theano_rng.binomial(size=h1_mean.shape, n=1, p=h1_mean, - dtype=theano.config.floatX) - return [pre_sigmoid_h1, h1_mean, h1_sample] - - def propdown(self, hid): - '''This function propagates the hidden units activation downwards to - the visible units - - Note that we return also the pre_sigmoid_activation of the layer. As - it will turn out later, due to how Theano deals with optimization and - stability this symbolic variable will be needed to write down a more - stable graph (see details in the reconstruction cost function) - ''' - pre_sigmoid_activation = T.dot(hid, self.W.T) + self.vbias - return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)] - - def sample_v_given_h(self, h0_sample): - ''' This function infers state of visible units given hidden units ''' - # compute the activation of the visible given the hidden sample - pre_sigmoid_v1, v1_mean = self.propdown(h0_sample) - # get a sample of the visible given their activation - # Note that theano_rng.binomial returns a symbolic sample of dtype - # int64 by default. If we want to keep our computations in floatX - # for the GPU we need to specify to return the dtype floatX - v1_sample = self.theano_rng.binomial(size=v1_mean.shape,n=1, p=v1_mean, - dtype=theano.config.floatX) - return [pre_sigmoid_v1, v1_mean, v1_sample] +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.sample_v_given_h We can then use these functions to define the symbolic graph for a Gibbs sampling step. We define two functions: @@ -452,22 +339,11 @@ sampling step. We define two functions: The code is as follows: -.. code-block:: python - - def gibbs_hvh(self, h0_sample): - ''' This function implements one step of Gibbs sampling, - starting from the hidden state''' - pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h0_sample) - pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample) - return [pre_sigmoid_v1, v1_mean, v1_sample, pre_sigmoid_h1, h1_mean, h1_sample] - - def gibbs_vhv(self, v0_sample): - ''' This function implements one step of Gibbs sampling, - starting from the visible state''' - pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v0_sample) - pre_sigmoid_v1, v1_mean, v1_sample = self.sample_v_given_h(h1_sample) - return [pre_sigmoid_h1, h1_mean, h1_sample, pre_sigmoid_v1, v1_mean, v1_sample] +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.gibbs_hvh +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.gibbs_vhv Note that we also return the pre-sigmoid activation. To understand why this is so you need to understand a bit about @@ -495,51 +371,15 @@ The class also has a function that computes the free energy of the model, needed for computing the gradient of the parameters (see Eq. :eq:`free_energy_grad`). Note that we also return the pre-sigmoid -.. code-block:: python - - - def free_energy(self, v_sample): - ''' Function to compute the free energy ''' - wx_b = T.dot(v_sample, self.W) + self.hbias - vbias_term = T.dot(v_sample, self.vbias) - hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1) - return -hidden_term - vbias_term - +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.free_energy We then add a ``get_cost_updates`` method, whose purpose is to generate the symbolic gradients for CD-k and PCD-k updates. -.. code-block:: python - - def get_cost_updates(self, lr=0.1, persistent=None, k=1): - """ - This functions implements one step of CD-k or PCD-k - - :param lr: learning rate used to train the RBM - - :param persistent: None for CD. For PCD, shared variable containing old state - of Gibbs chain. This must be a shared variable of size (batch size, number of - hidden units). - - :param k: number of Gibbs steps to do in CD-k/PCD-k - - Returns a proxy for the cost and the updates dictionary. The - dictionary contains the update rules for weights and biases but - also an update of the shared variable used to store the persistent - chain, if one is used. - """ - - # compute positive phase - pre_sigmoid_ph, ph_mean, ph_sample = self.sample_h_given_v(self.input) - - # decide how to initialize persistent chain: - # for CD, we use the newly generate hidden sample - # for PCD, we initialize from the old state of the chain - if persistent is None: - chain_start = ph_sample - else: - chain_start = persistent - +.. literalinclude:: ../code/rbm.py + :start-after: def gibbs_vhv + :end-before: # perform actual negative phase Note that ``get_cost_updates`` takes as argument a variable called ``persistent``. This allows us to use the same code to implement both CD and PCD. To use PCD, ``persistent`` should refer to a shared variable which contains the @@ -551,22 +391,9 @@ starting point of the chain, we can then compute the sample at the end of the Gibbs chain, sample that we need for getting the gradient (see Eq. :eq:`free_energy_grad`). To do so, we will use the ``scan`` op provided by Theano, therefore we urge the reader to look it up by following this `link `_. -.. code-block:: python - - # perform actual negative phase - # in order to implement CD-k/PCD-k we need to scan over the - # function that implements one gibbs step k times. - # Read Theano tutorial on scan for more information : - # http://deeplearning.net/software/theano/library/scan.html - # the scan will return the entire Gibbs chain - [pre_sigmoid_nvs, nv_means, nv_samples, pre_sigmoid_nhs, nh_means, nh_samples], updates = \ - theano.scan(self.gibbs_hvh, - # the None are place holders, saying that - # chain_start is the initial state corresponding to the - # 6th output - outputs_info=[None, None, None, None, None, chain_start], - n_steps=k) - +.. literalinclude:: ../code/rbm.py + :start-after: chain_start = persistent + :end-before: # determine gradients on RBM parameters Once we have the generated the chain we take the sample at the end of the chain to get the free energy of the negative phase. Note that the @@ -577,38 +404,18 @@ want (it will mess up our gradients) and therefore we need to indicate to ``T.grad`` that ``chain_end`` is a constant. We do this by using the argument ``consider_constant`` of ``T.grad``. -.. code-block:: python - - - # determine gradients on RBM parameters - # note that we only need the sample at the end of the chain - chain_end = nv_samples[-1] - - cost = T.mean(self.free_energy(self.input)) - T.mean(self.free_energy(chain_end)) - # We must not compute the gradient through the gibbs sampling - gparams = T.grad(cost, self.params, consider_constant=[chain_end]) +.. literalinclude:: ../code/rbm.py + :start-after: # determine gradients on RBM parameters + :end-before: # constructs the update dictionary Finally, we add to the updates dictionary returned by scan (which contains updates rules for random states of ``theano_rng``) to contain the parameter updates. In the case of PCD, these should also update the shared variable containing the state of the Gibbs chain. -.. code-block:: python - - # constructs the update dictionary - for gparam, param in zip(gparams, self.params): - # make sure that the learning rate is of the right dtype - updates[param] = param - gparam * T.cast(lr, dtype=theano.config.floatX) - if persistent: - # Note that this works only if persistent is a shared variable - updates[persistent] = nh_samples[-1] - # pseudo-likelihood is a better proxy for PCD - monitoring_cost = self.get_pseudo_likelihood_cost(updates) - else: - # reconstruction cross-entropy is a better proxy for CD - monitoring_cost = self.get_reconstruction_cost(updates, pre_sigmoid_nvs[-1]) - - return monitoring_cost, updates +.. literalinclude:: ../code/rbm.py + :start-after: # constructs the update dictionary + :end-before: def get_pseudo_likelihood_cost Tracking Progress ----------------- @@ -680,36 +487,8 @@ Note that for CD training the cost-entropy cost between the input and the reconstruction( the same as the one used for the de-noising autoencoder) is more reliable then the pseudo-loglikelihood. Here is the code we use to compute the pseudo-likelihood: -.. code-block:: python - - - def get_pseudo_likelihood_cost(self, updates): - """Stochastic approximation to the pseudo-likelihood""" - - # index of bit i in expression p(x_i | x_{\i}) - bit_i_idx = theano.shared(value=0, name='bit_i_idx') - - # binarize the input image by rounding to nearest integer - xi = T.iround(self.input) - - # calculate free energy for the given bit configuration - fe_xi = self.free_energy(xi) - - # flip bit x_i of matrix xi and preserve all other bits x_{\i} - # Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx], but assigns - # the result to xi_flip, instead of working in place on xi. - xi_flip = T.set_subtensor(xi[:, bit_i_idx], 1 - xi[:, bit_i_idx]) - - # calculate free energy with bit flipped - fe_xi_flip = self.free_energy(xi_flip) - - # equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i}))) - cost = T.mean(self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi))) - - # increment bit_i_idx % number as part of updates - updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible - - return cost +.. literalinclude:: ../code/rbm.py + :pyobject: RBM.get_pseudo_likelihood_cost Main Loop --------- @@ -729,44 +508,9 @@ Having these utility functions, we can start training the RBM and plot/save the filters after each training epoch. We train the RBM using PCD, as it has been shown to lead to a better generative model ([Tieleman08]_). -.. code-block:: python - - # it is ok for a theano function to have no output - # the purpose of train_rbm is solely to update the RBM parameters - train_rbm = theano.function([index], cost, - updates=updates, - givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size]}) - - plotting_time = 0. - start_time = time.clock() - - - # go through training epochs - for epoch in xrange(training_epochs): - - # go through the training set - mean_cost = [] - for batch_index in xrange(n_train_batches): - mean_cost += [train_rbm(batch_index)] - - print 'Training epoch %d, cost is '%epoch, numpy.mean(mean_cost) - - # Plot filters after each training epoch - plotting_start = time.clock() - # Construct image from the weight matrix - image = Image.fromarray(tile_raster_images( - X=rbm.W.get_value(borrow=True).T, - img_shape=(28, 28), tile_shape=(10, 10), - tile_spacing=(1, 1))) - image.save('filters_at_epoch_%i.png'%epoch) - plotting_stop = time.clock() - plotting_time += (plotting_stop - plotting_start) - - end_time = time.clock() - - pretraining_time = (end_time - start_time) - plotting_time - - print ('Training took %f minutes' % (pretraining_time / 60.)) +.. literalinclude:: ../code/rbm.py + :start-after: os.chdir(output_folder) + :end-before: Sampling from the RBM Once the RBM is trained, we can then use the ``gibbs_vhv`` function to implement the Gibbs chain required for sampling. We initialize the Gibbs chain starting @@ -775,20 +519,9 @@ in order to speed up convergence and avoid problems with random initialization. We again use Theano's ``scan`` op to do 1000 steps before each plotting. -.. code-block:: python - - ################################# - # Sampling from the RBM # - ################################# - - # find out the number of test samples - number_of_test_samples = test_set_x.get_value(borrow=True).shape[0] - - # pick random test examples, with which to initialize the persistent chain - test_idx = rng.randint(number_of_test_samples - 20) - persistent_vis_chain = theano.shared(numpy.asarray( - test_set_x.get_value(borrow=True)[test_idx: test_idx + 20], - dtype=theano.config.floatX)) +.. literalinclude:: ../code/rbm.py + :start-after: Sampling from the RBM + :end-before: plot_every = 1000 Next we create the 20 persistent chains in parallel to get our samples. To do so, we compile a theano function which performs one Gibbs step @@ -796,56 +529,9 @@ and updates the state of the persistent chain with the new visible sample. We apply this function iteratively for a large number of steps, plotting the samples at every 1000 steps. -.. code-block:: python - - - # find out the number of test - number_of_test_samples = test_set_x.get_value(borrow=True).shape[0] - - # pick random test examples, with which to initialize the persistent chain - test_idx = rng.randint(number_of_test_samples-n_chains) - persistent_vis_chain = theano.shared(numpy.array( - test_set_x.get_value(borrow=True)[test_idx:test_idx + 100], - dtype=theano.config.floatX)) - - plot_every = 1000 - # define one step of Gibbs sampling (mf = mean-field) - # define a function that does `plot_every` steps before returning the sample for plotting - [presig_hids, hid_mfs, hid_samples, presig_vis, vis_mfs, vis_samples], updates = \ - theano.scan(rbm.gibbs_vhv, - outputs_info=[None, None, None, None, None, persistent_vis_chain], - n_steps=plot_every) - - # add to updates the shared variable that takes care of our persistent - # chain : - updates.update({persistent_vis_chain: vis_samples[-1]}) - # construct the function that implements our persistent chain - # we generate the "mean field" activations for plotting and the actual samples for - # reinitializing the state of our persistent chain - sample_fn = theano.function([], [vis_mfs[-1], vis_samples[-1]], - updates=updates) - - # sample the RBM, plotting at least `n_samples` - n_samples = 10 - # create a space to store the image for plotting ( we need to leave - # room for the tile_spacing as well) - image_data = numpy.zeros((29 * n_samples + 1, 29 * n_chains - 1), - dtype='uint8') - for idx in xrange(n_samples): - # generate `plot_every` intermediate samples that we discard, because successive samples in the chain are too correlated - vis_mf, vis_sample = sample_fn() - image_data[29 * idx: 29 * idx + 28, :] = tile_raster_images( - X=vis_mf, - img_shape=(28, 28), - tile_shape=(1, batch_size), - tile_spacing=(1, 1)) - # construct image - - image = Image.fromarray(image_data) - print ' ... plotting sample ', idx - image.save('samples.png') - - +.. literalinclude:: ../code/rbm.py + :start-after: find out the number of test + :end-before: os.chdir('../') Results +++++++ diff --git a/doc/rnnrbm.txt b/doc/rnnrbm.txt index ea95e177..d64a0c4a 100644 --- a/doc/rnnrbm.txt +++ b/doc/rnnrbm.txt @@ -81,59 +81,8 @@ The RBM layer The ``build_rbm`` function shown below builds a Gibbs chain from an input mini-batch (a binary matrix) via the CD approximation. Note that it also supports a single frame (a binary vector) in the non-batch case. - -.. code-block:: python - - def build_rbm(v, W, bv, bh, k): - '''Construct a k-step Gibbs chain starting at v for an RBM. - - v : Theano vector or matrix - If a matrix, multiple chains will be run in parallel (batch). - W : Theano matrix - Weight matrix of the RBM. - bv : Theano vector - Visible bias vector of the RBM. - bh : Theano vector - Hidden bias vector of the RBM. - k : scalar or Theano scalar - Length of the Gibbs chain. - - Return a (v_sample, cost, monitor, updates) tuple: - - v_sample : Theano vector or matrix with the same shape as `v` - Corresponds to the generated sample(s). - cost : Theano scalar - Expression whose gradient with respect to W, bv, bh is the CD-k approximation - to the log-likelihood of `v` (training example) under the RBM. - The cost is averaged in the batch case. - monitor: Theano scalar - Pseudo log-likelihood (also averaged in the batch case). - updates: dictionary of Theano variable -> Theano variable - The `updates` object returned by scan.''' - - def gibbs_step(v): - mean_h = T.nnet.sigmoid(T.dot(v, W) + bh) - h = rng.binomial(size=mean_h.shape, n=1, p=mean_h, - dtype=theano.config.floatX) - mean_v = T.nnet.sigmoid(T.dot(h, W.T) + bv) - v = rng.binomial(size=mean_v.shape, n=1, p=mean_v, - dtype=theano.config.floatX) - return mean_v, v - - chain, updates = theano.scan(lambda v: gibbs_step(v)[1], outputs_info=[v], - n_steps=k) - v_sample = chain[-1] - - mean_v = gibbs_step(v_sample)[0] - monitor = T.xlogx.xlogy0(v, mean_v) + T.xlogx.xlogy0(1 - v, 1 - mean_v) - monitor = monitor.sum() / v.shape[0] - - def free_energy(v): - return -(v * bv).sum() - T.log(1 + T.exp(T.dot(v, W) + bh)).sum() - cost = (free_energy(v) - free_energy(v_sample)) / v.shape[0] - - return v_sample, cost, monitor, updates - +.. literalinclude:: ../code/rnnrbm.py + :pyobject: build_rbm The RNN layer --------------- @@ -141,202 +90,16 @@ The RNN layer The ``build_rnnrbm`` function defines the RNN recurrence relation to obtain the RBM parameters; the recurrence function is flexible enough to serve both in the training scenario where :math:`v^{(t)}` is given and the "batch" RBM is constructed at the end on the whole sequence at once, and in the generation scenario where :math:`v^{(t)}` is sampled separately at each time step using the Gibbs chain defined above. -.. code-block:: python - - def build_rnnrbm(n_visible, n_hidden, n_hidden_recurrent): - '''Construct a symbolic RNN-RBM and initialize parameters. - - n_visible : integer - Number of visible units. - n_hidden : integer - Number of hidden units of the conditional RBMs. - n_hidden_recurrent : integer - Number of hidden units of the RNN. - - Return a (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) tuple: - - v : Theano matrix - Symbolic variable holding an input sequence (used during training) - v_sample : Theano matrix - Symbolic variable holding the negative particles for CD log-likelihood - gradient estimation (used during training) - cost : Theano scalar - Expression whose gradient (considering v_sample constant) corresponds to the - LL gradient of the RNN-RBM (used during training) - monitor : Theano scalar - Frame-level pseudo-likelihood (useful for monitoring during training) - params : tuple of Theano shared variables - The parameters of the model to be optimized during training. - updates_train : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - training function. - v_t : Theano matrix - Symbolic variable holding a generated sequence (used during sampling) - updates_generate : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - generation function.''' - - W = shared_normal(n_visible, n_hidden, 0.01) - bv = shared_zeros(n_visible) - bh = shared_zeros(n_hidden) - Wuh = shared_normal(n_hidden_recurrent, n_hidden, 0.0001) - Wuv = shared_normal(n_hidden_recurrent, n_visible, 0.0001) - Wvu = shared_normal(n_visible, n_hidden_recurrent, 0.0001) - Wuu = shared_normal(n_hidden_recurrent, n_hidden_recurrent, 0.0001) - bu = shared_zeros(n_hidden_recurrent) - - params = W, bv, bh, Wuh, Wuv, Wvu, Wuu, bu # learned parameters as shared - # variables - - v = T.matrix() # a training sequence - u0 = T.zeros((n_hidden_recurrent,)) # initial value for the RNN hidden - # units - - # If `v_t` is given, deterministic recurrence to compute the variable - # biases bv_t, bh_t at each time step. If `v_t` is None, same recurrence - # but with a separate Gibbs chain at each time step to sample (generate) - # from the RNN-RBM. The resulting sample v_t is returned in order to be - # passed down to the sequence history. - def recurrence(v_t, u_tm1): - bv_t = bv + T.dot(u_tm1, Wuv) - bh_t = bh + T.dot(u_tm1, Wuh) - generate = v_t is None - if generate: - v_t, _, _, updates = build_rbm(T.zeros((n_visible,)), W, bv_t, - bh_t, k=25) - u_t = T.tanh(bu + T.dot(v_t, Wvu) + T.dot(u_tm1, Wuu)) - return ([v_t, u_t], updates) if generate else [u_t, bv_t, bh_t] - - # For training, the deterministic recurrence is used to compute all the - # {bv_t, bh_t, 1 <= t <= T} given v. Conditional RBMs can then be trained - # in batches using those parameters. - (u_t, bv_t, bh_t), updates_train = theano.scan( - lambda v_t, u_tm1, *_: recurrence(v_t, u_tm1), - sequences=v, outputs_info=[u0, None, None], non_sequences=params) - v_sample, cost, monitor, updates_rbm = build_rbm(v, W, bv_t[:], bh_t[:], - k=15) - updates_train.update(updates_rbm) - - # symbolic loop for sequence generation - (v_t, u_t), updates_generate = theano.scan( - lambda u_tm1, *_: recurrence(None, u_tm1), - outputs_info=[None, u0], non_sequences=params, n_steps=200) - - return (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) - +.. literalinclude:: ../code/rnnrbm.py + :pyobject: build_rnnrbm Putting it all together --------------------------- We now have all the necessary ingredients to start training our network on real symbolic sequences of polyphonic music. -.. code-block:: python - - class RnnRbm: - '''Simple class to train an RNN-RBM from MIDI files and to generate sample - sequences.''' - - def __init__(self, n_hidden=150, n_hidden_recurrent=100, lr=0.001, - r=(21, 109), dt=0.3): - '''Constructs and compiles Theano functions for training and sequence - generation. - - n_hidden : integer - Number of hidden units of the conditional RBMs. - n_hidden_recurrent : integer - Number of hidden units of the RNN. - lr : float - Learning rate - r : (integer, integer) tuple - Specifies the pitch range of the piano-roll in MIDI note numbers, including - r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the - RBM at a given time step. The default (21, 109) corresponds to the full range - of piano (88 notes). - dt : float - Sampling period when converting the MIDI files into piano-rolls, or - equivalently the time difference between consecutive time steps.''' - - self.r = r - self.dt = dt - (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) = build_rnnrbm(r[1] - r[0], n_hidden, - n_hidden_recurrent) - - gradient = T.grad(cost, params, consider_constant=[v_sample]) - updates_train.update(((p, p - lr * g) for p, g in zip(params, - gradient))) - self.train_function = theano.function([v], monitor, - updates=updates_train) - self.generate_function = theano.function([], v_t, - updates=updates_generate) - - def train(self, files, batch_size=100, num_epochs=200): - '''Train the RNN-RBM via stochastic gradient descent (SGD) using MIDI - files converted to piano-rolls. - - files : list of strings - List of MIDI files that will be loaded as piano-rolls for training. - batch_size : integer - Training sequences will be split into subsequences of at most this size - before applying the SGD updates. - num_epochs : integer - Number of epochs (pass over the training set) performed. The user can - safely interrupt training with Ctrl+C at any time.''' - - assert len(files) > 0, 'Training set is empty!' \ - ' (did you download the data files?)' - dataset = [midiread(f, self.r, - self.dt).piano_roll.astype(theano.config.floatX) - for f in files] - - try: - for epoch in xrange(num_epochs): - numpy.random.shuffle(dataset) - costs = [] - - for s, sequence in enumerate(dataset): - for i in xrange(0, len(sequence), batch_size): - cost = self.train_function(sequence[i:i + batch_size]) - costs.append(cost) - - print 'Epoch %i/%i' % (epoch + 1, num_epochs), - print numpy.mean(costs) - sys.stdout.flush() - - except KeyboardInterrupt: - print 'Interrupted by user.' - - def generate(self, filename, show=True): - '''Generate a sample sequence, plot the resulting piano-roll and save - it as a MIDI file. - - filename : string - A MIDI file will be created at this location. - show : boolean - If True, a piano-roll of the generated sequence will be shown.''' - - piano_roll = self.generate_function() - midiwrite(filename, piano_roll, self.r, self.dt) - if show: - extent = (0, self.dt * len(piano_roll)) + self.r - pylab.figure() - pylab.imshow(piano_roll.T, origin='lower', aspect='auto', - interpolation='nearest', cmap=pylab.cm.gray_r, - extent=extent) - pylab.xlabel('time (s)') - pylab.ylabel('MIDI note number') - pylab.title('generated piano-roll') - - - if __name__ == '__main__': - model = RnnRbm() - model.train(glob.glob('../data/Nottingham/train/*.mid')) - model.generate('sample1.mid') - model.generate('sample2.mid') - pylab.show() - +.. literalinclude:: ../code/rnnrbm.py + :pyobject: RnnRbm Results ++++++++ diff --git a/issues_open/6_benchmarking_pybrain.txt b/issues_open/6_benchmarking_pybrain.txt index 52753eaf..45540bf1 100644 --- a/issues_open/6_benchmarking_pybrain.txt +++ b/issues_open/6_benchmarking_pybrain.txt @@ -43,7 +43,7 @@ Observations : RESULTS : - logreg on maggie46 + logistic_sgd on maggie46 Total error: 0.015611011103 Total error: 0.00966772673335 From 3f4613534db5de49ff496b3bd64f1a5bbb633bdb Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 16:27:48 -0700 Subject: [PATCH 049/417] Ran code through pep8 and fixed all occurrences of E251 unexpected spaces around keyword / parameter equals. Fixed indentation of comments in rnnrbm.py. --- code/SdA.py | 38 ++++++++-------- code/convolutional_mlp.py | 60 ++++++++++++------------- code/dA.py | 40 ++++++++--------- code/logistic_sgd.py | 44 +++++++++--------- code/mlp.py | 48 ++++++++++---------- code/rbm.py | 40 ++++++++--------- code/rnnrbm.py | 94 +++++++++++++++++++-------------------- 7 files changed, 182 insertions(+), 182 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index 26927d3d..e7edbde6 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -59,11 +59,11 @@ class SdA(object): def __init__(self, numpy_rng, - theano_rng = None, - n_ins = 784, - hidden_layers_sizes = [500, 500], - n_outs = 10, - corruption_levels = [0.1, 0.1] + theano_rng=None, + n_ins=784, + hidden_layers_sizes=[500, 500], + n_outs=10, + corruption_levels=[0.1, 0.1] ): """ This class is made to support a variable number of layers. @@ -246,9 +246,9 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): (test_set_x, test_set_y) = datasets[2] # compute number of minibatches for training, validation and testing - n_valid_batches = valid_set_x.get_value(borrow = True).shape[0] + n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] n_valid_batches /= batch_size - n_test_batches = test_set_x.get_value(borrow = True).shape[0] + n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_test_batches /= batch_size index = T.lscalar('index') # index to a [mini]batch @@ -263,30 +263,30 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): ] train_fn = theano.function( - inputs = [index], - outputs = self.finetune_cost, - updates = updates, - givens = { + inputs=[index], + outputs=self.finetune_cost, + updates=updates, + givens={ self.x: train_set_x[index * batch_size : (index + 1) * batch_size], self.y: train_set_y[index * batch_size : (index + 1) * batch_size] }, - name = 'train' + name='train' ) test_score_i = theano.function( [index], self.errors, - givens = { + givens={ self.x: test_set_x[index * batch_size : (index + 1) * batch_size], self.y: test_set_y[index * batch_size : (index + 1) * batch_size] }, - name = 'test' + name='test' ) valid_score_i = theano.function( [index], self.errors, - givens = { + givens={ self.x: valid_set_x[index * batch_size : (index + 1) * batch_size], self.y: valid_set_y[index * batch_size : (index + 1) * batch_size] }, @@ -345,10 +345,10 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, print '... building the model' # construct the stacked denoising autoencoder class sda = SdA( - numpy_rng = numpy_rng, - n_ins = 28 * 28, - hidden_layers_sizes = [1000, 1000, 1000], - n_outs = 10 + numpy_rng=numpy_rng, + n_ins=28 * 28, + hidden_layers_sizes=[1000, 1000, 1000], + n_outs=10 ) ######################### diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index dbcbfa69..a662f4b3 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -78,10 +78,10 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared( numpy.asarray( - rng.uniform(low = -W_bound, high = W_bound, size = filter_shape), - dtype = theano.config.floatX + rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), + dtype=theano.config.floatX ), - borrow = True + borrow=True ) # the bias is a 1D tensor -- one bias per output feature map @@ -90,17 +90,17 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): # convolve input feature maps with filters conv_out = conv.conv2d( - input = input, - filters = self.W, - filter_shape = filter_shape, - image_shape = image_shape + input=input, + filters=self.W, + filter_shape=filter_shape, + image_shape=image_shape ) # downsample each feature map individually, using maxpooling pooled_out = downsample.max_pool_2d( - input = conv_out, - ds = poolsize, - ignore_border = True + input=conv_out, + ds=poolsize, + ignore_border=True ) # add the bias term. Since the bias is a vector (1D array), we first @@ -141,9 +141,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow = True).shape[0] - n_valid_batches = valid_set_x.get_value(borrow = True).shape[0] - n_test_batches = test_set_x.get_value(borrow = True).shape[0] + n_train_batches = train_set_x.get_value(borrow=True).shape[0] + n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] + n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_train_batches /= batch_size n_valid_batches /= batch_size n_test_batches /= batch_size @@ -171,10 +171,10 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12) layer0 = LeNetConvPoolLayer( rng, - input = layer0_input, - image_shape = (batch_size, 1, 28, 28), - filter_shape = (nkerns[0], 1, 5, 5), - poolsize = (2, 2) + input=layer0_input, + image_shape=(batch_size, 1, 28, 28), + filter_shape=(nkerns[0], 1, 5, 5), + poolsize=(2, 2) ) # Construct the second convolutional pooling layer @@ -183,10 +183,10 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4) layer1 = LeNetConvPoolLayer( rng, - input = layer0.output, - image_shape = (batch_size, nkerns[0], 12, 12), - filter_shape = (nkerns[1], nkerns[0], 5, 5), - poolsize = (2, 2) + input=layer0.output, + image_shape=(batch_size, nkerns[0], 12, 12), + filter_shape=(nkerns[1], nkerns[0], 5, 5), + poolsize=(2, 2) ) # the HiddenLayer being fully-connected, it operates on 2D matrices of @@ -197,14 +197,14 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # construct a fully-connected sigmoidal layer layer2 = HiddenLayer( rng, - input = layer2_input, - n_in = nkerns[1] * 4 * 4, - n_out = 500, - activation = T.tanh + input=layer2_input, + n_in=nkerns[1] * 4 * 4, + n_out=500, + activation=T.tanh ) # classify the values of the fully-connected sigmoidal layer - layer3 = LogisticRegression(input = layer2.output, n_in = 500, n_out = 10) + layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10) # the cost we minimize during training is the NLL of the model cost = layer3.negative_log_likelihood(y) @@ -213,7 +213,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, test_model = theano.function( [index], layer3.errors(y), - givens = { + givens={ x: test_set_x[index * batch_size : (index + 1) * batch_size], y: test_set_y[index * batch_size : (index + 1) * batch_size] } @@ -222,7 +222,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, validate_model = theano.function( [index], layer3.errors(y), - givens = { + givens={ x: valid_set_x[index * batch_size : (index + 1) * batch_size], y: valid_set_y[index * batch_size : (index + 1) * batch_size] } @@ -247,8 +247,8 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, train_model = theano.function( [index], cost, - updates = updates, - givens = { + updates=updates, + givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size], y: train_set_y[index * batch_size : (index + 1) * batch_size] } diff --git a/code/dA.py b/code/dA.py index f3ae58b2..803963d2 100644 --- a/code/dA.py +++ b/code/dA.py @@ -78,13 +78,13 @@ class dA(object): def __init__( self, numpy_rng, - theano_rng = None, - input = None, - n_visible = 784, - n_hidden = 500, - W = None, - bhid = None, - bvis = None + theano_rng=None, + input=None, + n_visible=784, + n_hidden=500, + W=None, + bhid=None, + bvis=None ): """ Initialize the dA class by specifying the number of visible units (the @@ -287,11 +287,11 @@ def test_dA(learning_rate=0.1, training_epochs=15, theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( - numpy_rng = rng, - theano_rng = theano_rng, - input = x, - n_visible = 28 * 28, - n_hidden = 500 + numpy_rng=rng, + theano_rng=theano_rng, + input=x, + n_visible=28 * 28, + n_hidden=500 ) cost, updates = da.get_cost_updates( @@ -302,8 +302,8 @@ def test_dA(learning_rate=0.1, training_epochs=15, train_da = theano.function( [index], cost, - updates = updates, - givens = { + updates=updates, + givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size] } ) @@ -344,15 +344,15 @@ def test_dA(learning_rate=0.1, training_epochs=15, theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( - numpy_rng = rng, - theano_rng = theano_rng, - input = x, - n_visible = 28 * 28, - n_hidden = 500 + numpy_rng=rng, + theano_rng=theano_rng, + input=x, + n_visible=28 * 28, + n_hidden=500 ) cost, updates = da.get_cost_updates( - corruption_level = 0.3, + corruption_level=0.3, learning_rate=learning_rate ) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index be6c4358..4dfee1ac 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -75,21 +75,21 @@ def __init__(self, input, n_in, n_out): # initialize with 0 the weights W as a matrix of shape (n_in, n_out) self.W = theano.shared( - value = numpy.zeros( + value=numpy.zeros( (n_in, n_out), - dtype = theano.config.floatX + dtype=theano.config.floatX ), - name = 'W', - borrow = True + name='W', + borrow=True ) # initialize the baises b as a vector of n_out 0s self.b = theano.shared( - value = numpy.zeros( + value=numpy.zeros( (n_out,), - dtype = theano.config.floatX + dtype=theano.config.floatX ), - name = 'b', - borrow = True + name='b', + borrow=True ) # symbolic expression for computing the matrix of class-membership probabilities @@ -101,7 +101,7 @@ def __init__(self, input, n_in, n_out): # symbolic description of how to compute prediction as class whose probability # is maximal - self.y_pred = T.argmax(self.p_y_given_x, axis = 1) + self.y_pred = T.argmax(self.p_y_given_x, axis=1) # parameters of the model self.params = [self.W, self.b] @@ -277,7 +277,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # construct the logistic regression class # Each MNIST image has size 28*28 - classifier = LogisticRegression(input = x, n_in = 28 * 28, n_out = 10) + classifier = LogisticRegression(input=x, n_in=28 * 28, n_out=10) # the cost we minimize during training is the negative log likelihood of # the model in symbolic format @@ -286,26 +286,26 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # compiling a Theano function that computes the mistakes that are made by # the model on a minibatch test_model = theano.function( - inputs = [index], - outputs = classifier.errors(y), - givens = { + inputs=[index], + outputs=classifier.errors(y), + givens={ x: test_set_x[index * batch_size : (index + 1) * batch_size], y: test_set_y[index * batch_size : (index + 1) * batch_size] } ) validate_model = theano.function( - inputs = [index], - outputs = classifier.errors(y), - givens = { + inputs=[index], + outputs=classifier.errors(y), + givens={ x: valid_set_x[index * batch_size : (index + 1) * batch_size], y: valid_set_y[index * batch_size : (index + 1) * batch_size] } ) # compute the gradient of cost with respect to theta = (W,b) - g_W = T.grad(cost = cost, wrt = classifier.W) - g_b = T.grad(cost = cost, wrt = classifier.b) + g_W = T.grad(cost=cost, wrt=classifier.W) + g_b = T.grad(cost=cost, wrt=classifier.b) # specify how to update the parameters of the model as a list of # (variable, update expression) pairs. @@ -316,10 +316,10 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # the same time updates the parameter of the model based on the rules # defined in `updates` train_model = theano.function( - inputs = [index], - outputs = cost, - updates = updates, - givens = { + inputs=[index], + outputs=cost, + updates=updates, + givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size], y: train_set_y[index * batch_size : (index + 1) * batch_size] } diff --git a/code/mlp.py b/code/mlp.py index 32cad444..2d0effe1 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -81,20 +81,20 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, if W is None: W_values = numpy.asarray( rng.uniform( - low = -numpy.sqrt(6. / (n_in + n_out)), - high = numpy.sqrt(6. / (n_in + n_out)), - size = (n_in, n_out) + low=-numpy.sqrt(6. / (n_in + n_out)), + high=numpy.sqrt(6. / (n_in + n_out)), + size=(n_in, n_out) ), - dtype = theano.config.floatX + dtype=theano.config.floatX ) if activation == theano.tensor.nnet.sigmoid: W_values *= 4 - W = theano.shared(value = W_values, name = 'W', borrow = True) + W = theano.shared(value=W_values, name='W', borrow=True) if b is None: - b_values = numpy.zeros((n_out,), dtype = theano.config.floatX) - b = theano.shared(value = b_values, name = 'b', borrow = True) + b_values = numpy.zeros((n_out,), dtype=theano.config.floatX) + b = theano.shared(value=b_values, name='b', borrow=True) self.W = W self.b = b @@ -147,19 +147,19 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): # LogisticRegression layer; the activation function can be replaced by # sigmoid or any other nonlinear function self.hiddenLayer = HiddenLayer( - rng = rng, - input = input, - n_in = n_in, - n_out = n_hidden, - activation = T.tanh + rng=rng, + input=input, + n_in=n_in, + n_out=n_hidden, + activation=T.tanh ) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegression( - input = self.hiddenLayer.output, - n_in = n_hidden, - n_out = n_out + input=self.hiddenLayer.output, + n_in=n_hidden, + n_out=n_out ) # L1 norm ; one regularization option is to enforce L1 norm to @@ -239,11 +239,11 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # construct the MLP class classifier = MLP( - rng = rng, - input = x, - n_in = 28 * 28, - n_hidden = n_hidden, - n_out = 10 + rng=rng, + input=x, + n_in=28 * 28, + n_hidden=n_hidden, + n_out=10 ) # the cost we minimize during training is the negative log likelihood of @@ -287,10 +287,10 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # in the same time updates the parameter of the model based on the rules # defined in `updates` train_model = theano.function( - inputs = [index], - outputs = cost, - updates = updates, - givens = { + inputs=[index], + outputs=cost, + updates=updates, + givens={ x: train_set_x[index * batch_size : (index + 1) * batch_size], y: train_set_y[index * batch_size : (index + 1) * batch_size] } diff --git a/code/rbm.py b/code/rbm.py index c9f779da..5de8caac 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -29,14 +29,14 @@ class RBM(object): """Restricted Boltzmann Machine (RBM) """ def __init__( self, - input = None, - n_visible = 784, - n_hidden = 500, - W = None, - hbias = None, - vbias = None, - numpy_rng = None, - theano_rng = None + input=None, + n_visible=784, + n_hidden=500, + W=None, + hbias=None, + vbias=None, + numpy_rng=None, + theano_rng=None ): """ RBM constructor. Defines the parameters of the model along with @@ -80,35 +80,35 @@ def __init__( # that the code is runable on GPU initial_W = numpy.asarray( numpy_rng.uniform( - low = -4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high = 4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size = (n_visible, n_hidden) + low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), + high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), + size=(n_visible, n_hidden) ), - dtype = theano.config.floatX + dtype=theano.config.floatX ) # theano shared variables for weights and biases - W = theano.shared(value = initial_W, name = 'W', borrow = True) + W = theano.shared(value=initial_W, name='W', borrow=True) if hbias is None: # create shared variable for hidden units bias hbias = theano.shared( - value = numpy.zeros( + value=numpy.zeros( n_hidden, - dtype = theano.config.floatX + dtype=theano.config.floatX ), - name = 'hbias', + name='hbias', borrow=True ) if vbias is None: # create shared variable for visible units bias vbias = theano.shared( - value = numpy.zeros( + value=numpy.zeros( n_visible, - dtype = theano.config.floatX + dtype=theano.config.floatX ), - name = 'vbias', - borrow = True + name='vbias', + borrow=True ) # initialize input layer for standalone RBM or layer0 of DBN diff --git a/code/rnnrbm.py b/code/rnnrbm.py index 9cef86c8..6d0bf931 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -29,28 +29,28 @@ def build_rbm(v, W, bv, bh, k): '''Construct a k-step Gibbs chain starting at v for an RBM. v : Theano vector or matrix - If a matrix, multiple chains will be run in parallel (batch). + If a matrix, multiple chains will be run in parallel (batch). W : Theano matrix - Weight matrix of the RBM. + Weight matrix of the RBM. bv : Theano vector - Visible bias vector of the RBM. + Visible bias vector of the RBM. bh : Theano vector - Hidden bias vector of the RBM. + Hidden bias vector of the RBM. k : scalar or Theano scalar - Length of the Gibbs chain. + Length of the Gibbs chain. Return a (v_sample, cost, monitor, updates) tuple: v_sample : Theano vector or matrix with the same shape as `v` - Corresponds to the generated sample(s). + Corresponds to the generated sample(s). cost : Theano scalar - Expression whose gradient with respect to W, bv, bh is the CD-k approximation - to the log-likelihood of `v` (training example) under the RBM. - The cost is averaged in the batch case. + Expression whose gradient with respect to W, bv, bh is the CD-k approximation + to the log-likelihood of `v` (training example) under the RBM. + The cost is averaged in the batch case. monitor: Theano scalar - Pseudo log-likelihood (also averaged in the batch case). + Pseudo log-likelihood (also averaged in the batch case). updates: dictionary of Theano variable -> Theano variable - The `updates` object returned by scan.''' + The `updates` object returned by scan.''' def gibbs_step(v): mean_h = T.nnet.sigmoid(T.dot(v, W) + bh) @@ -92,35 +92,35 @@ def build_rnnrbm(n_visible, n_hidden, n_hidden_recurrent): '''Construct a symbolic RNN-RBM and initialize parameters. n_visible : integer - Number of visible units. + Number of visible units. n_hidden : integer - Number of hidden units of the conditional RBMs. + Number of hidden units of the conditional RBMs. n_hidden_recurrent : integer - Number of hidden units of the RNN. + Number of hidden units of the RNN. Return a (v, v_sample, cost, monitor, params, updates_train, v_t, updates_generate) tuple: v : Theano matrix - Symbolic variable holding an input sequence (used during training) + Symbolic variable holding an input sequence (used during training) v_sample : Theano matrix - Symbolic variable holding the negative particles for CD log-likelihood - gradient estimation (used during training) + Symbolic variable holding the negative particles for CD log-likelihood + gradient estimation (used during training) cost : Theano scalar - Expression whose gradient (considering v_sample constant) corresponds to the - LL gradient of the RNN-RBM (used during training) + Expression whose gradient (considering v_sample constant) corresponds to the + LL gradient of the RNN-RBM (used during training) monitor : Theano scalar - Frame-level pseudo-likelihood (useful for monitoring during training) + Frame-level pseudo-likelihood (useful for monitoring during training) params : tuple of Theano shared variables - The parameters of the model to be optimized during training. + The parameters of the model to be optimized during training. updates_train : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - training function. + Update object that should be passed to theano.function when compiling the + training function. v_t : Theano matrix - Symbolic variable holding a generated sequence (used during sampling) + Symbolic variable holding a generated sequence (used during sampling) updates_generate : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - generation function.''' + Update object that should be passed to theano.function when compiling the + generation function.''' W = shared_normal(n_visible, n_hidden, 0.01) bv = shared_zeros(n_visible) @@ -178,29 +178,29 @@ class RnnRbm: def __init__( self, - n_hidden = 150, - n_hidden_recurrent = 100, - lr = 0.001, - r = (21, 109), - dt = 0.3 + n_hidden=150, + n_hidden_recurrent=100, + lr=0.001, + r=(21, 109), + dt=0.3 ): '''Constructs and compiles Theano functions for training and sequence generation. n_hidden : integer - Number of hidden units of the conditional RBMs. + Number of hidden units of the conditional RBMs. n_hidden_recurrent : integer - Number of hidden units of the RNN. + Number of hidden units of the RNN. lr : float - Learning rate + Learning rate r : (integer, integer) tuple - Specifies the pitch range of the piano-roll in MIDI note numbers, including - r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the - RBM at a given time step. The default (21, 109) corresponds to the full range - of piano (88 notes). + Specifies the pitch range of the piano-roll in MIDI note numbers, including + r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the + RBM at a given time step. The default (21, 109) corresponds to the full range + of piano (88 notes). dt : float - Sampling period when converting the MIDI files into piano-rolls, or - equivalently the time difference between consecutive time steps.''' + Sampling period when converting the MIDI files into piano-rolls, or + equivalently the time difference between consecutive time steps.''' self.r = r self.dt = dt @@ -221,13 +221,13 @@ def train(self, files, batch_size=100, num_epochs=200): files converted to piano-rolls. files : list of strings - List of MIDI files that will be loaded as piano-rolls for training. + List of MIDI files that will be loaded as piano-rolls for training. batch_size : integer - Training sequences will be split into subsequences of at most this size - before applying the SGD updates. + Training sequences will be split into subsequences of at most this size + before applying the SGD updates. num_epochs : integer - Number of epochs (pass over the training set) performed. The user can - safely interrupt training with Ctrl+C at any time.''' + Number of epochs (pass over the training set) performed. The user can + safely interrupt training with Ctrl+C at any time.''' assert len(files) > 0, 'Training set is empty!' \ ' (did you download the data files?)' @@ -257,9 +257,9 @@ def generate(self, filename, show=True): it as a MIDI file. filename : string - A MIDI file will be created at this location. + A MIDI file will be created at this location. show : boolean - If True, a piano-roll of the generated sequence will be shown.''' + If True, a piano-roll of the generated sequence will be shown.''' piano_roll = self.generate_function() midiwrite(filename, piano_roll, self.r, self.dt) From 7312feb56a056aa245732675b470f18db011ba75 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 16:31:29 -0700 Subject: [PATCH 050/417] Ran code through pep8 and fixed all occurrences of E124. --- code/SdA.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/SdA.py b/code/SdA.py index e7edbde6..86c7a884 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -57,7 +57,8 @@ class SdA(object): the dAs are only used to initialize the weights. """ - def __init__(self, + def __init__( + self, numpy_rng, theano_rng=None, n_ins=784, From 3007d07ee95b0819fced327c8204fc5f93076fd9 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 16:36:30 -0700 Subject: [PATCH 051/417] Replaced final (programmatic) reference to lenet. --- doc/convolutional_mlp.txt | 2 +- doc/intro.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/convolutional_mlp.txt b/doc/convolutional_mlp.txt index df4c7b32..dc91ab2d 100644 --- a/doc/convolutional_mlp.txt +++ b/doc/convolutional_mlp.txt @@ -1,4 +1,4 @@ -.. _lenet: +.. _convolutional_mlp: Convolutional Neural Networks (LeNet) ===================================== diff --git a/doc/intro.txt b/doc/intro.txt index 8573272b..7011e917 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -31,7 +31,7 @@ The purely supervised learning algorithms are meant to be read in order: #. :ref:`Logistic Regression ` - using Theano for something simple #. :ref:`Multilayer perceptron ` - introduction to layers - #. :ref:`Deep Convolutional Network ` - a simplified version of LeNet5 + #. :ref:`Deep Convolutional Network ` - a simplified version of LeNet5 The unsupervised and semi-supervised learning algorithms can be read in any order (the auto-encoders can be read independently of the RBM/DBN thread): From 750673ced26f3128e538c061ce3858cba8ef3cd0 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 17:02:26 -0700 Subject: [PATCH 052/417] Restored doc/convolutional_mlp.txt to doc/lenet.txt and doc/logistic_sgd.txt to doc/logreg.txt to respect existing bookmarks. --- doc/DBN.txt | 4 ++-- doc/SdA.txt | 4 ++-- doc/contents.txt | 4 ++-- doc/dA.txt | 2 +- doc/intro.txt | 4 ++-- doc/{convolutional_mlp.txt => lenet.txt} | 6 +++--- doc/{logistic_sgd.txt => logreg.txt} | 2 +- doc/mlp.txt | 6 +++--- doc/rbm.txt | 2 +- 9 files changed, 17 insertions(+), 17 deletions(-) rename doc/{convolutional_mlp.txt => lenet.txt} (99%) rename doc/{logistic_sgd.txt => logreg.txt} (99%) diff --git a/doc/DBN.txt b/doc/DBN.txt index 017585eb..bc113fc0 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -4,7 +4,7 @@ Deep Belief Networks ==================== .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd` + This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp` and :doc:`rbm`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the @@ -164,7 +164,7 @@ hidden bias with its corresponding sigmoid layer. All that is left is to stack one last logistic regression layer in order to form an MLP. We will use the ``LogisticRegression`` class introduced in -:ref:`logistic_sgd`. +:ref:`logreg`. .. literalinclude:: ../code/DBN.py :start-after: # We now need to add a logistic layer on top of the MLP diff --git a/doc/SdA.txt b/doc/SdA.txt index cf012342..91926bd9 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -4,7 +4,7 @@ Stacked Denoising Autoencoders (SdA) ==================================== .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd` + This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. @@ -95,7 +95,7 @@ bias of the encoding part with its corresponding sigmoid layer. All we need now is to add the logistic layer on top of the sigmoid layers such that we have an MLP. We will -use the ``LogisticRegression`` class introduced in :ref:`logistic_sgd`. +use the ``LogisticRegression`` class introduced in :ref:`logreg`. .. literalinclude:: ../code/SdA.py :start-after: self.dA_layers.append(dA_layer) diff --git a/doc/contents.txt b/doc/contents.txt index 3bd48785..381043d9 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -11,9 +11,9 @@ Contents LICENSE intro gettingstarted - logistic_sgd + logreg mlp - convolutional_mlp + lenet dA SdA rbm diff --git a/doc/dA.txt b/doc/dA.txt index 1c59dfa2..00902398 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -4,7 +4,7 @@ Denoising Autoencoders (dA) =========================== .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd` + This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. diff --git a/doc/intro.txt b/doc/intro.txt index 7011e917..19ab4bc7 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -29,9 +29,9 @@ read through our :ref:`gettingstarted` chapter -- it introduces the notation, an The purely supervised learning algorithms are meant to be read in order: - #. :ref:`Logistic Regression ` - using Theano for something simple + #. :ref:`Logistic Regression ` - using Theano for something simple #. :ref:`Multilayer perceptron ` - introduction to layers - #. :ref:`Deep Convolutional Network ` - a simplified version of LeNet5 + #. :ref:`Deep Convolutional Network ` - a simplified version of LeNet5 The unsupervised and semi-supervised learning algorithms can be read in any order (the auto-encoders can be read independently of the RBM/DBN thread): diff --git a/doc/convolutional_mlp.txt b/doc/lenet.txt similarity index 99% rename from doc/convolutional_mlp.txt rename to doc/lenet.txt index dc91ab2d..d827c834 100644 --- a/doc/convolutional_mlp.txt +++ b/doc/lenet.txt @@ -1,10 +1,10 @@ -.. _convolutional_mlp: +.. _lenet: Convolutional Neural Networks (LeNet) ===================================== .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd` and + This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally, it uses the following new Theano functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `floatX`_, `downsample`_ , `conv2d`_, `dimshuffle`_. If you intend to run the @@ -406,7 +406,7 @@ layer. Notice that when initializing the weight values, the fan-in is determined by the size of the receptive fields and the number of input feature maps. -Finally, using the LogisticRegression class defined in :doc:`logistic_sgd` and +Finally, using the LogisticRegression class defined in :doc:`logreg` and the HiddenLayer class defined in :doc:`mlp` , we can instantiate the network as follows. diff --git a/doc/logistic_sgd.txt b/doc/logreg.txt similarity index 99% rename from doc/logistic_sgd.txt rename to doc/logreg.txt index 65956210..8e41dac3 100644 --- a/doc/logistic_sgd.txt +++ b/doc/logreg.txt @@ -1,6 +1,6 @@ .. index:: Logistic Regression -.. _logistic_sgd : +.. _logreg : Classifying MNIST digits using Logistic Regression diff --git a/doc/mlp.txt b/doc/mlp.txt index 03884286..7da0ffbd 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -7,7 +7,7 @@ Multilayer Perceptron ===================== .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd`. + This section assumes the reader has already read through :doc:`logreg`. Additionally, it uses the following new Theano functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, :ref:`L1_L2_regularization`, `floatX`_. If you intend to run the @@ -80,7 +80,7 @@ extension to vectors and tensors consists in applying them element-wise The output vector is then obtained as: :math:`o(x) = G(b^{(2)} + W^{(2)} h(x))`. The reader should recognize the form we already used for -:doc:`logistic_sgd`. As before, +:doc:`logreg`. As before, class-membership probabilities can be obtained by choosing :math:`G` as the :math:`softmax` function (in the case of multi-class classification). @@ -129,7 +129,7 @@ to use something else. If you look into theory this class implements the graph that computes the hidden layer value :math:`h(x) = \Phi(x) = s(b^{(1)} + W^{(1)} x)`. If you give this as input to the ``LogisticRegression`` class, -implemented in the previous tutorial :doc:`logistic_sgd`, you get the output +implemented in the previous tutorial :doc:`logreg`, you get the output of the MLP. You can see this in the following short implementation of the ``MLP`` class. diff --git a/doc/rbm.txt b/doc/rbm.txt index a2a962aa..2ccd80f3 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -5,7 +5,7 @@ Restricted Boltzmann Machines (RBM) .. note:: - This section assumes the reader has already read through :doc:`logistic_sgd` + This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_ and `scan`_. If you intend to run the code on GPU also read `GPU`_. From cef1016e4678147d24c554f4cde0bd285ef6c102 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 17:20:47 -0700 Subject: [PATCH 053/417] Fixed :start-after: options to respect lack of spaces around parameter ='s. --- doc/logreg.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/logreg.txt b/doc/logreg.txt index 8e41dac3..ec6b4bfc 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -158,7 +158,7 @@ The last step involves defining a (symbolic) cost variable to minimize, using the instance method ``classifier.negative_log_likelihood``. .. literalinclude:: ../code/logistic_sgd.py - :start-after: classifier = LogisticRegression(input = x, n_in = 28 * 28, n_out = 10) + :start-after: classifier = LogisticRegression(input=x, n_in=28 * 28, n_out=10) :end-before: # compiling a Theano function that computes the mistakes Note how x is an implicit symbolic input to the symbolic definition of cost, @@ -191,7 +191,7 @@ computation graph. Performing one-step of gradient descent can then be done as follows: .. literalinclude:: ../code/logistic_sgd.py - :start-after: g_b = T.grad(cost = cost, wrt = classifier.b) + :start-after: g_b = T.grad(cost=cost, wrt=classifier.b) :end-before: # TRAIN MODEL The ``updates`` list contains, for each parameter, the From e3f9cd21f29c310f6f4121e05b09e74ac740a6ec Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Thu, 9 Oct 2014 22:46:28 -0700 Subject: [PATCH 054/417] Minimized lines > 79 characters, trailing whitespace, and whitespace before : --- code/SdA.py | 38 ++++++++++++++++++++----------- code/cA.py | 2 +- code/convolutional_mlp.py | 47 +++++++++++++++++++++------------------ code/dA.py | 30 ++++++++++++------------- code/logistic_sgd.py | 40 ++++++++++++++++++++------------- code/mlp.py | 24 +++++++++++--------- code/rbm.py | 16 ++++++------- code/rnnrbm.py | 46 ++++++++++++++++++++------------------ 8 files changed, 136 insertions(+), 107 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index 86c7a884..a742761b 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -58,11 +58,11 @@ class SdA(object): """ def __init__( - self, - numpy_rng, - theano_rng=None, + self, + numpy_rng, + theano_rng=None, n_ins=784, - hidden_layers_sizes=[500, 500], + hidden_layers_sizes=[500, 500], n_outs=10, corruption_levels=[0.1, 0.1] ): @@ -268,28 +268,40 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): outputs=self.finetune_cost, updates=updates, givens={ - self.x: train_set_x[index * batch_size : (index + 1) * batch_size], - self.y: train_set_y[index * batch_size : (index + 1) * batch_size] + self.x: train_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: train_set_y[ + index * batch_size: (index + 1) * batch_size + ] }, name='train' ) test_score_i = theano.function( - [index], + [index], self.errors, givens={ - self.x: test_set_x[index * batch_size : (index + 1) * batch_size], - self.y: test_set_y[index * batch_size : (index + 1) * batch_size] + self.x: test_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: test_set_y[ + index * batch_size: (index + 1) * batch_size + ] }, name='test' ) valid_score_i = theano.function( - [index], + [index], self.errors, givens={ - self.x: valid_set_x[index * batch_size : (index + 1) * batch_size], - self.y: valid_set_y[index * batch_size : (index + 1) * batch_size] + self.x: valid_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: valid_set_y[ + index * batch_size: (index + 1) * batch_size + ] }, name='valid' ) @@ -346,7 +358,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, print '... building the model' # construct the stacked denoising autoencoder class sda = SdA( - numpy_rng=numpy_rng, + numpy_rng=numpy_rng, n_ins=28 * 28, hidden_layers_sizes=[1000, 1000, 1000], n_outs=10 diff --git a/code/cA.py b/code/cA.py index 986a2d2a..dbe3f9f3 100644 --- a/code/cA.py +++ b/code/cA.py @@ -12,7 +12,7 @@ squared Frobenius norm of the Jacobian of the hidden mapping h with respect to the visible units yields the contractive auto-encoder: - - \sum_{k=1}^d[ x_k \log z_k + (1-x_k) \log( 1-z_k)] + \| \frac{\partial h(x)}{\partial x} \|^2 + - \sum_{k=1}^d[ x_k \log z_k + (1-x_k) \log( 1-z_k)] + \| \frac{\partial h(x)}{\partial x} \|^2 References : - S. Rifai, P. Vincent, X. Muller, X. Glorot, Y. Bengio: Contractive diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index a662f4b3..f8117eb2 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -90,16 +90,16 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): # convolve input feature maps with filters conv_out = conv.conv2d( - input=input, + input=input, filters=self.W, - filter_shape=filter_shape, + filter_shape=filter_shape, image_shape=image_shape ) # downsample each feature map individually, using maxpooling pooled_out = downsample.max_pool_2d( input=conv_out, - ds=poolsize, + ds=poolsize, ignore_border=True ) @@ -170,7 +170,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # maxpooling reduces this further to (24/2,24/2) = (12,12) # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12) layer0 = LeNetConvPoolLayer( - rng, + rng, input=layer0_input, image_shape=(batch_size, 1, 28, 28), filter_shape=(nkerns[0], 1, 5, 5), @@ -182,10 +182,10 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # maxpooling reduces this further to (8/2,8/2) = (4,4) # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4) layer1 = LeNetConvPoolLayer( - rng, + rng, input=layer0.output, image_shape=(batch_size, nkerns[0], 12, 12), - filter_shape=(nkerns[1], nkerns[0], 5, 5), + filter_shape=(nkerns[1], nkerns[0], 5, 5), poolsize=(2, 2) ) @@ -196,10 +196,10 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # construct a fully-connected sigmoidal layer layer2 = HiddenLayer( - rng, - input=layer2_input, + rng, + input=layer2_input, n_in=nkerns[1] * 4 * 4, - n_out=500, + n_out=500, activation=T.tanh ) @@ -211,20 +211,20 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # create a function to compute the mistakes that are made by the model test_model = theano.function( - [index], + [index], layer3.errors(y), givens={ - x: test_set_x[index * batch_size : (index + 1) * batch_size], - y: test_set_y[index * batch_size : (index + 1) * batch_size] + x: test_set_x[index * batch_size: (index + 1) * batch_size], + y: test_set_y[index * batch_size: (index + 1) * batch_size] } ) validate_model = theano.function( - [index], + [index], layer3.errors(y), givens={ - x: valid_set_x[index * batch_size : (index + 1) * batch_size], - y: valid_set_y[index * batch_size : (index + 1) * batch_size] + x: valid_set_x[index * batch_size: (index + 1) * batch_size], + y: valid_set_y[index * batch_size: (index + 1) * batch_size] } ) @@ -245,12 +245,12 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, ] train_model = theano.function( - [index], - cost, + [index], + cost, updates=updates, givens={ - x: train_set_x[index * batch_size : (index + 1) * batch_size], - y: train_set_y[index * batch_size : (index + 1) * batch_size] + x: train_set_x[index * batch_size: (index + 1) * batch_size], + y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) @@ -312,10 +312,13 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, best_iter = iter # test it on the test set - test_losses = [test_model(i) for i in xrange(n_test_batches)] + test_losses = [ + test_model(i) + for i in xrange(n_test_batches) + ] test_score = numpy.mean(test_losses) - print((' epoch %i, minibatch %i/%i, test error of best ' - 'model %f %%') % + print((' epoch %i, minibatch %i/%i, test error of ' + 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) diff --git a/code/dA.py b/code/dA.py index 803963d2..e883f455 100644 --- a/code/dA.py +++ b/code/dA.py @@ -76,14 +76,14 @@ class dA(object): """ def __init__( - self, - numpy_rng, - theano_rng=None, + self, + numpy_rng, + theano_rng=None, input=None, - n_visible=784, + n_visible=784, n_hidden=500, - W=None, - bhid=None, + W=None, + bhid=None, bvis=None ): """ @@ -287,10 +287,10 @@ def test_dA(learning_rate=0.1, training_epochs=15, theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( - numpy_rng=rng, - theano_rng=theano_rng, + numpy_rng=rng, + theano_rng=theano_rng, input=x, - n_visible=28 * 28, + n_visible=28 * 28, n_hidden=500 ) @@ -300,11 +300,11 @@ def test_dA(learning_rate=0.1, training_epochs=15, ) train_da = theano.function( - [index], - cost, + [index], + cost, updates=updates, givens={ - x: train_set_x[index * batch_size : (index + 1) * batch_size] + x: train_set_x[index * batch_size: (index + 1) * batch_size] } ) @@ -344,10 +344,10 @@ def test_dA(learning_rate=0.1, training_epochs=15, theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( - numpy_rng=rng, - theano_rng=theano_rng, + numpy_rng=rng, + theano_rng=theano_rng, input=x, - n_visible=28 * 28, + n_visible=28 * 28, n_hidden=500 ) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 4dfee1ac..62788b80 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -79,7 +79,7 @@ def __init__(self, input, n_in, n_out): (n_in, n_out), dtype=theano.config.floatX ), - name='W', + name='W', borrow=True ) # initialize the baises b as a vector of n_out 0s @@ -88,19 +88,22 @@ def __init__(self, input, n_in, n_out): (n_out,), dtype=theano.config.floatX ), - name='b', + name='b', borrow=True ) - # symbolic expression for computing the matrix of class-membership probabilities + # symbolic expression for computing the matrix of class-membership + # probabilities # Where: - # W is a matrix where column-k represent the separation hyper plain for class-k + # W is a matrix where column-k represent the separation hyper plain for + # class-k # x is a matrix where row-j represents input training sample-j - # b is a vector where element-k represent the free parameter of hyper plain-k + # b is a vector where element-k represent the free parameter of hyper + # plain-k self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b) - # symbolic description of how to compute prediction as class whose probability - # is maximal + # symbolic description of how to compute prediction as class whose + # probability is maximal self.y_pred = T.argmax(self.p_y_given_x, axis=1) # parameters of the model @@ -173,13 +176,20 @@ def load_data(dataset): data_dir, data_file = os.path.split(dataset) if data_dir == "" and not os.path.isfile(dataset): # Check if dataset is in the data directory. - new_path = os.path.join(os.path.split(__file__)[0], "..", "data", dataset) + new_path = os.path.join( + os.path.split(__file__)[0], + "..", + "data", + dataset + ) if os.path.isfile(new_path) or data_file == 'mnist.pkl.gz': dataset = new_path if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz': import urllib - origin = 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz' + origin = ( + 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz' + ) print 'Downloading data from %s' % origin urllib.urlretrieve(origin, dataset) @@ -289,8 +299,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, inputs=[index], outputs=classifier.errors(y), givens={ - x: test_set_x[index * batch_size : (index + 1) * batch_size], - y: test_set_y[index * batch_size : (index + 1) * batch_size] + x: test_set_x[index * batch_size: (index + 1) * batch_size], + y: test_set_y[index * batch_size: (index + 1) * batch_size] } ) @@ -298,8 +308,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, inputs=[index], outputs=classifier.errors(y), givens={ - x: valid_set_x[index * batch_size : (index + 1) * batch_size], - y: valid_set_y[index * batch_size : (index + 1) * batch_size] + x: valid_set_x[index * batch_size: (index + 1) * batch_size], + y: valid_set_y[index * batch_size: (index + 1) * batch_size] } ) @@ -320,8 +330,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, outputs=cost, updates=updates, givens={ - x: train_set_x[index * batch_size : (index + 1) * batch_size], - y: train_set_y[index * batch_size : (index + 1) * batch_size] + x: train_set_x[index * batch_size: (index + 1) * batch_size], + y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) diff --git a/code/mlp.py b/code/mlp.py index 2d0effe1..20e8c748 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -84,7 +84,7 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, low=-numpy.sqrt(6. / (n_in + n_out)), high=numpy.sqrt(6. / (n_in + n_out)), size=(n_in, n_out) - ), + ), dtype=theano.config.floatX ) if activation == theano.tensor.nnet.sigmoid: @@ -147,9 +147,9 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): # LogisticRegression layer; the activation function can be replaced by # sigmoid or any other nonlinear function self.hiddenLayer = HiddenLayer( - rng=rng, + rng=rng, input=input, - n_in=n_in, + n_in=n_in, n_out=n_hidden, activation=T.tanh ) @@ -175,7 +175,9 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer - self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood + self.negative_log_likelihood = ( + self.logRegressionLayer.negative_log_likelihood + ) # same holds for the function computing the number of errors self.errors = self.logRegressionLayer.errors @@ -239,10 +241,10 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # construct the MLP class classifier = MLP( - rng=rng, - input=x, + rng=rng, + input=x, n_in=28 * 28, - n_hidden=n_hidden, + n_hidden=n_hidden, n_out=10 ) @@ -279,7 +281,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # is a pair formed from the two lists : # C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)] updates = [ - (param, param - learning_rate * gparam) + (param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams) ] @@ -287,12 +289,12 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # in the same time updates the parameter of the model based on the rules # defined in `updates` train_model = theano.function( - inputs=[index], + inputs=[index], outputs=cost, updates=updates, givens={ - x: train_set_x[index * batch_size : (index + 1) * batch_size], - y: train_set_y[index * batch_size : (index + 1) * batch_size] + x: train_set_x[index * batch_size: (index + 1) * batch_size], + y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) diff --git a/code/rbm.py b/code/rbm.py index 5de8caac..064d7b37 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -28,13 +28,13 @@ class RBM(object): """Restricted Boltzmann Machine (RBM) """ def __init__( - self, - input=None, - n_visible=784, + self, + input=None, + n_visible=784, n_hidden=500, - W=None, - hbias=None, - vbias=None, + W=None, + hbias=None, + vbias=None, numpy_rng=None, theano_rng=None ): @@ -96,7 +96,7 @@ def __init__( n_hidden, dtype=theano.config.floatX ), - name='hbias', + name='hbias', borrow=True ) @@ -107,7 +107,7 @@ def __init__( n_visible, dtype=theano.config.floatX ), - name='vbias', + name='vbias', borrow=True ) diff --git a/code/rnnrbm.py b/code/rnnrbm.py index 6d0bf931..42f04d8c 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -11,7 +11,9 @@ try: import pylab except ImportError: - print "pylab isn't available, if you use their fonctionality, it will crash" + print ( + "pylab isn't available. If you use its functionality, it will crash." + ) print "It can be installed with 'pip install -q Pillow'" from midi.utils import midiread, midiwrite @@ -44,9 +46,9 @@ def build_rbm(v, W, bv, bh, k): v_sample : Theano vector or matrix with the same shape as `v` Corresponds to the generated sample(s). cost : Theano scalar - Expression whose gradient with respect to W, bv, bh is the CD-k approximation - to the log-likelihood of `v` (training example) under the RBM. - The cost is averaged in the batch case. + Expression whose gradient with respect to W, bv, bh is the CD-k + approximation to the log-likelihood of `v` (training example) under the + RBM. The cost is averaged in the batch case. monitor: Theano scalar Pseudo log-likelihood (also averaged in the batch case). updates: dictionary of Theano variable -> Theano variable @@ -107,20 +109,20 @@ def build_rnnrbm(n_visible, n_hidden, n_hidden_recurrent): Symbolic variable holding the negative particles for CD log-likelihood gradient estimation (used during training) cost : Theano scalar - Expression whose gradient (considering v_sample constant) corresponds to the - LL gradient of the RNN-RBM (used during training) + Expression whose gradient (considering v_sample constant) corresponds + to the LL gradient of the RNN-RBM (used during training) monitor : Theano scalar Frame-level pseudo-likelihood (useful for monitoring during training) params : tuple of Theano shared variables The parameters of the model to be optimized during training. updates_train : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - training function. + Update object that should be passed to theano.function when compiling + the training function. v_t : Theano matrix Symbolic variable holding a generated sequence (used during sampling) updates_generate : dictionary of Theano variable -> Theano variable - Update object that should be passed to theano.function when compiling the - generation function.''' + Update object that should be passed to theano.function when compiling + the generation function.''' W = shared_normal(n_visible, n_hidden, 0.01) bv = shared_zeros(n_visible) @@ -177,11 +179,11 @@ class RnnRbm: sequences.''' def __init__( - self, - n_hidden=150, - n_hidden_recurrent=100, + self, + n_hidden=150, + n_hidden_recurrent=100, lr=0.001, - r=(21, 109), + r=(21, 109), dt=0.3 ): '''Constructs and compiles Theano functions for training and sequence @@ -194,10 +196,10 @@ def __init__( lr : float Learning rate r : (integer, integer) tuple - Specifies the pitch range of the piano-roll in MIDI note numbers, including - r[0] but not r[1], such that r[1]-r[0] is the number of visible units of the - RBM at a given time step. The default (21, 109) corresponds to the full range - of piano (88 notes). + Specifies the pitch range of the piano-roll in MIDI note numbers, + including r[0] but not r[1], such that r[1]-r[0] is the number of + visible units of the RBM at a given time step. The default (21, + 109) corresponds to the full range of piano (88 notes). dt : float Sampling period when converting the MIDI files into piano-rolls, or equivalently the time difference between consecutive time steps.''' @@ -223,11 +225,11 @@ def train(self, files, batch_size=100, num_epochs=200): files : list of strings List of MIDI files that will be loaded as piano-rolls for training. batch_size : integer - Training sequences will be split into subsequences of at most this size - before applying the SGD updates. + Training sequences will be split into subsequences of at most this + size before applying the SGD updates. num_epochs : integer - Number of epochs (pass over the training set) performed. The user can - safely interrupt training with Ctrl+C at any time.''' + Number of epochs (pass over the training set) performed. The user + can safely interrupt training with Ctrl+C at any time.''' assert len(files) > 0, 'Training set is empty!' \ ' (did you download the data files?)' From 7d867f02ee3b7a5490a24c5cbef3173b02ff6649 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Fri, 10 Oct 2014 17:51:46 -0700 Subject: [PATCH 055/417] Used start-snippet-# and end-snippet-# for precise inclusion of code in ReStructuredText files. Many cosmetic changes to reduce pep8 warnings. --- code/DBN.py | 115 ++++++++++++++++++++----------- code/SdA.py | 69 ++++++++++++------- code/cA.py | 27 +++++--- code/convolutional_mlp.py | 10 +-- code/dA.py | 55 ++++++++++----- code/hmc/hmc.py | 37 +++++----- code/logistic_cg.py | 110 ++++++++++++++++++------------ code/logistic_sgd.py | 57 +++++++++++----- code/mlp.py | 80 +++++++++++++++------- code/rbm.py | 140 ++++++++++++++++++++++++++------------ code/rnnrbm.py | 26 ++++--- code/utils.py | 12 ++-- doc/DBN.txt | 8 +-- doc/SdA.txt | 18 ++--- doc/dA.txt | 3 +- doc/hmc.txt | 30 +++++++- doc/lenet.txt | 4 +- doc/logreg.txt | 12 ++-- doc/mlp.txt | 21 ++++-- doc/rbm.txt | 31 +++++---- 20 files changed, 564 insertions(+), 301 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 4a287dd4..eacbbee4 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -17,6 +17,7 @@ from rbm import RBM +# start-snippet-1 class DBN(object): """Deep Belief Network @@ -65,7 +66,7 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector # of [int] labels - + # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first # construct the DBN as a deep multilayer perceptron, and when @@ -174,12 +175,14 @@ def pretraining_functions(self, train_set_x, batch_size, k): persistent=None, k=k) # compile the theano function - fn = theano.function(inputs=[index, - theano.Param(learning_rate, default=0.1)], - outputs=cost, - updates=updates, - givens={self.x: - train_set_x[batch_begin:batch_end]}) + fn = theano.function( + inputs=[index, theano.Param(learning_rate, default=0.1)], + outputs=cost, + updates=updates, + givens={ + self.x: train_set_x[batch_begin:batch_end] + } + ) # append `fn` to the list of functions pretrain_fns.append(fn) @@ -224,25 +227,45 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): for param, gparam in zip(self.params, gparams): updates.append((param, param - gparam * learning_rate)) - train_fn = theano.function(inputs=[index], - outputs=self.finetune_cost, - updates=updates, - givens={self.x: train_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: train_set_y[index * batch_size: - (index + 1) * batch_size]}) - - test_score_i = theano.function([index], self.errors, - givens={self.x: test_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: test_set_y[index * batch_size: - (index + 1) * batch_size]}) - - valid_score_i = theano.function([index], self.errors, - givens={self.x: valid_set_x[index * batch_size: - (index + 1) * batch_size], - self.y: valid_set_y[index * batch_size: - (index + 1) * batch_size]}) + train_fn = theano.function( + inputs=[index], + outputs=self.finetune_cost, + updates=updates, + givens={ + self.x: train_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: train_set_y[ + index * batch_size: (index + 1) * batch_size + ] + } + ) + + test_score_i = theano.function( + [index], + self.errors, + givens={ + self.x: test_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: test_set_y[ + index * batch_size: (index + 1) * batch_size + ] + } + ) + + valid_score_i = theano.function( + [index], + self.errors, + givens={ + self.x: valid_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: valid_set_y[ + index * batch_size: (index + 1) * batch_size + ] + } + ) # Create a function that scans the entire validation set def valid_score(): @@ -296,6 +319,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, hidden_layers_sizes=[1000, 1000, 1000], n_outs=10) + # start-snippet-2 ######################### # PRETRAINING THE MODEL # ######################### @@ -319,10 +343,10 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, print numpy.mean(c) end_time = time.clock() + # end-snippet-2 print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)) - ######################## # FINETUNING THE MODEL # ######################## @@ -330,10 +354,12 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, # get the training, validation and testing function for the model print '... getting the finetuning functions' train_fn, validate_model, test_model = dbn.build_finetune_functions( - datasets=datasets, batch_size=batch_size, - learning_rate=finetune_lr) + datasets=datasets, + batch_size=batch_size, + learning_rate=finetune_lr + ) - print '... finetunning the model' + print '... finetuning the model' # early-stopping parameters patience = 4 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is @@ -342,7 +368,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many - # minibatche before checking the network + # minibatches before checking the network # on the validation set; in this case we # check every epoch @@ -365,16 +391,24 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, validation_losses = validate_model() this_validation_loss = numpy.mean(validation_losses) - print('epoch %i, minibatch %i/%i, validation error %f %%' % \ - (epoch, minibatch_index + 1, n_train_batches, - this_validation_loss * 100.)) + print( + 'epoch %i, minibatch %i/%i, validation error %f %%' + % ( + epoch, + minibatch_index + 1, + n_train_batches, + this_validation_loss * 100. + ) + ) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough - if (this_validation_loss < best_validation_loss * - improvement_threshold): + if ( + this_validation_loss < best_validation_loss * + improvement_threshold + ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number @@ -394,9 +428,12 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, break end_time = time.clock() - print(('Optimization complete with best validation score of %f %%,' - 'with test performance %f %%') % - (best_validation_loss * 100., test_score * 100.)) + print( + ( + 'Optimization complete with best validation score of %f %%,' + 'with test performance %f %%' + ) % (best_validation_loss * 100., test_score * 100.) + ) print >> sys.stderr, ('The fine tuning code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) diff --git a/code/SdA.py b/code/SdA.py index a742761b..34d5678b 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -46,6 +46,7 @@ from dA import dA +# start-snippet-1 class SdA(object): """Stacked denoising auto-encoder class (SdA) @@ -104,6 +105,7 @@ def __init__( self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels + # end-snippet-1 # The SdA is an MLP, for which all weights of intermediate layers # are shared with a different denoising autoencoders @@ -115,6 +117,7 @@ def __init__( # During finetunining we will finish training the SdA by doing # stochastich gradient descent on the MLP + # start-snippet-2 for i in xrange(self.n_layers): # construct the sigmoidal layer @@ -157,11 +160,13 @@ def __init__( W=sigmoid_layer.W, bhid=sigmoid_layer.b) self.dA_layers.append(dA_layer) - + # end-snippet-2 # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticRegression( - input=self.sigmoid_layers[-1].output, - n_in=hidden_layers_sizes[-1], n_out=n_outs) + input=self.sigmoid_layers[-1].output, + n_in=hidden_layers_sizes[-1], + n_out=n_outs + ) self.params.extend(self.logLayer.params) # construct a function that implements one step of finetunining @@ -210,13 +215,18 @@ def pretraining_functions(self, train_set_x, batch_size): cost, updates = dA.get_cost_updates(corruption_level, learning_rate) # compile the theano function - fn = theano.function(inputs=[index, - theano.Param(corruption_level, default=0.2), - theano.Param(learning_rate, default=0.1)], - outputs=cost, - updates=updates, - givens={self.x: train_set_x[batch_begin: - batch_end]}) + fn = theano.function( + inputs=[ + index, + theano.Param(corruption_level, default=0.2), + theano.Param(learning_rate, default=0.1) + ], + outputs=cost, + updates=updates, + givens={ + self.x: train_set_x[batch_begin: batch_end] + } + ) # append `fn` to the list of functions pretrain_fns.append(fn) @@ -282,12 +292,12 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): [index], self.errors, givens={ - self.x: test_set_x[ - index * batch_size: (index + 1) * batch_size - ], - self.y: test_set_y[ - index * batch_size: (index + 1) * batch_size - ] + self.x: test_set_x[ + index * batch_size: (index + 1) * batch_size + ], + self.y: test_set_y[ + index * batch_size: (index + 1) * batch_size + ] }, name='test' ) @@ -354,6 +364,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, n_train_batches /= batch_size # numpy random generator + # start-snippet-3 numpy_rng = numpy.random.RandomState(89677) print '... building the model' # construct the stacked denoising autoencoder class @@ -363,7 +374,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, hidden_layers_sizes=[1000, 1000, 1000], n_outs=10 ) - + # end-snippet-3 start-snippet-4 ######################### # PRETRAINING THE MODEL # ######################### @@ -392,7 +403,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)) - + # end-snippet-4 ######################## # FINETUNING THE MODEL # ######################## @@ -400,8 +411,10 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, # get the training, validation and testing function for the model print '... getting the finetuning functions' train_fn, validate_model, test_model = sda.build_finetune_functions( - datasets=datasets, batch_size=batch_size, - learning_rate=finetune_lr) + datasets=datasets, + batch_size=batch_size, + learning_rate=finetune_lr + ) print '... finetunning the model' # early-stopping parameters @@ -441,8 +454,10 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough - if (this_validation_loss < best_validation_loss * - improvement_threshold): + if ( + this_validation_loss < best_validation_loss * + improvement_threshold + ): patience = max(patience, iter * patience_increase) # save best validation score and iteration number @@ -462,9 +477,13 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, break end_time = time.clock() - print(('Optimization complete with best validation score of %f %%,' - 'with test performance %f %%') % - (best_validation_loss * 100., test_score * 100.)) + print( + ( + 'Optimization complete with best validation score of %f %%,' + 'with test performance %f %%' + ) + % (best_validation_loss * 100., test_score * 100.) + ) print >> sys.stderr, ('The training code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % ((end_time - start_time) / 60.)) diff --git a/code/cA.py b/code/cA.py index dbe3f9f3..c4e7874a 100644 --- a/code/cA.py +++ b/code/cA.py @@ -131,11 +131,14 @@ def __init__(self, numpy_rng, input=None, n_visible=784, n_hidden=100, # 4*sqrt(6./(n_hidden+n_visible))the output of uniform if # converted using asarray to dtype # theano.config.floatX so that the code is runable on GPU - initial_W = numpy.asarray(numpy_rng.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), - dtype=theano.config.floatX) + initial_W = numpy.asarray( + numpy_rng.uniform( + low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), + high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), + size=(n_visible, n_hidden) + ), + dtype=theano.config.floatX + ) W = theano.shared(value=initial_W, name='W', borrow=True) if not bvis: @@ -186,7 +189,7 @@ def get_reconstructed_input(self, hidden): hidden layer """ - return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) + return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) def get_cost_updates(self, contraction_level, learning_rate): """ This function computes the cost and the updates for one trainng @@ -265,10 +268,14 @@ def test_cA(learning_rate=0.01, training_epochs=20, cost, updates = ca.get_cost_updates(contraction_level=contraction_level, learning_rate=learning_rate) - train_ca = theano.function([index], [T.mean(ca.L_rec), ca.L_jacob], - updates=updates, - givens={x: train_set_x[index * batch_size: - (index + 1) * batch_size]}) + train_ca = theano.function( + [index], + [T.mean(ca.L_rec), ca.L_jacob], + updates=updates, + givens={ + x: train_set_x[index * batch_size: (index + 1) * batch_size] + } + ) start_time = time.clock() diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index f8117eb2..895c54e9 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -150,10 +150,11 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch + + # start-snippet-1 x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels - ishape = (28, 28) # this is the size of MNIST images ###################### @@ -253,6 +254,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) + # end-snippet-1 ############### # TRAIN MODEL # @@ -295,8 +297,8 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, validation_losses = [validate_model(i) for i in xrange(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) - print('epoch %i, minibatch %i/%i, validation error %f %%' % \ - (epoch, minibatch_index + 1, n_train_batches, \ + print('epoch %i, minibatch %i/%i, validation error %f %%' % + (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) # if we got the best validation score until now @@ -328,7 +330,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, end_time = time.clock() print('Optimization complete.') - print('Best validation score of %f %% obtained at iteration %i,'\ + print('Best validation score of %f %% obtained at iteration %i,' 'with test performance %f %%' % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) print >> sys.stderr, ('The code for file ' + diff --git a/code/dA.py b/code/dA.py index e883f455..6344221a 100644 --- a/code/dA.py +++ b/code/dA.py @@ -51,6 +51,7 @@ import Image +# start-snippet-1 class dA(object): """Denoising Auto-Encoder class (dA) @@ -146,22 +147,34 @@ def __init__( # 4*sqrt(6./(n_hidden+n_visible))the output of uniform if # converted using asarray to dtype # theano.config.floatX so that the code is runable on GPU - initial_W = numpy.asarray(numpy_rng.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), dtype=theano.config.floatX) + initial_W = numpy.asarray( + numpy_rng.uniform( + low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), + high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), + size=(n_visible, n_hidden) + ), + dtype=theano.config.floatX + ) W = theano.shared(value=initial_W, name='W', borrow=True) if not bvis: - bvis = theano.shared(value=numpy.zeros(n_visible, - dtype=theano.config.floatX), - borrow=True) + bvis = theano.shared( + value=numpy.zeros( + n_visible, + dtype=theano.config.floatX + ), + borrow=True + ) if not bhid: - bhid = theano.shared(value=numpy.zeros(n_hidden, - dtype=theano.config.floatX), - name='b', - borrow=True) + bhid = theano.shared( + value=numpy.zeros( + n_hidden, + dtype=theano.config.floatX + ), + name='b', + borrow=True + ) self.W = W # b corresponds to the bias of the hidden @@ -180,6 +193,7 @@ def __init__( self.x = input self.params = [self.W, self.b, self.b_prime] + # end-snippet-1 def get_corrupted_input(self, input, corruption_level): """This function keeps ``1-corruption_level`` entries of the inputs the @@ -203,9 +217,9 @@ def get_corrupted_input(self, input, corruption_level): correctly as it only support float32 for now. """ - return self.theano_rng.binomial(size=input.shape, n=1, - p=1 - corruption_level, - dtype=theano.config.floatX) * input + return self.theano_rng.binomial(size=input.shape, n=1, + p=1 - corruption_level, + dtype=theano.config.floatX) * input def get_hidden_values(self, input): """ Computes the values of the hidden layer """ @@ -216,7 +230,7 @@ def get_reconstructed_input(self, hidden): hidden layer """ - return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) + return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) def get_cost_updates(self, corruption_level, learning_rate): """ This function computes the cost and the updates for one trainng @@ -356,9 +370,14 @@ def test_dA(learning_rate=0.1, training_epochs=15, learning_rate=learning_rate ) - train_da = theano.function([index], cost, updates=updates, - givens={x: train_set_x[index * batch_size: - (index + 1) * batch_size]}) + train_da = theano.function( + [index], + cost, + updates=updates, + givens={ + x: train_set_x[index * batch_size: (index + 1) * batch_size] + } + ) start_time = time.clock() diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index e134e02a..740e3719 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -171,7 +171,7 @@ def leapfrog(pos, vel, step): # return new proposal state return final_pos, final_vel - +# start-snippet-1 def hmc_move(s_rng, positions, energy_fn, stepsize, n_steps): """ This function performs one-step of Hybrid Monte-Carlo sampling. We start by @@ -202,27 +202,29 @@ def hmc_move(s_rng, positions, energy_fn, stepsize, n_steps): rval2: theano matrix Matrix whose rows contain the proposed "new position" """ - + # end-snippet-1 start-snippet-2 # sample random velocity initial_vel = s_rng.normal(size=positions.shape) - + # end-snippet-2 start-snippet-3 # perform simulation of particles subject to Hamiltonian dynamics final_pos, final_vel = simulate_dynamics( - initial_pos=positions, - initial_vel=initial_vel, - stepsize=stepsize, - n_steps=n_steps, - energy_fn=energy_fn) - + initial_pos=positions, + initial_vel=initial_vel, + stepsize=stepsize, + n_steps=n_steps, + energy_fn=energy_fn + ) + # end-snippet-3 start-snippet-4 # accept/reject the proposed move based on the joint distribution accept = metropolis_hastings_accept( - energy_prev=hamiltonian(positions, initial_vel, energy_fn), - energy_next=hamiltonian(final_pos, final_vel, energy_fn), - s_rng=s_rng) - + energy_prev=hamiltonian(positions, initial_vel, energy_fn), + energy_next=hamiltonian(final_pos, final_vel, energy_fn), + s_rng=s_rng + ) + # end-snippet-4 return accept, final_pos - +# start-snippet-5 def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, target_acceptance_rate, stepsize_inc, stepsize_dec, stepsize_min, stepsize_max, avg_acceptance_slowness): @@ -276,7 +278,7 @@ def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, accept_matrix = accept.dimshuffle(0, *(('x',) * (final_pos.ndim - 1))) # if accept is True, update to `final_pos` else stay put new_positions = TT.switch(accept_matrix, final_pos, positions) - + # end-snippet-5 start-snippet-7 ## STEPSIZE UPDATES ## # if acceptance rate is too low, our sampler is too "noisy" and we reduce # the stepsize. If it is too high, our sampler is too conservative, we can @@ -286,17 +288,18 @@ def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, # maintain stepsize in [stepsize_min, stepsize_max] new_stepsize = TT.clip(_new_stepsize, stepsize_min, stepsize_max) + # end-snippet-7 start-snippet-6 ## ACCEPT RATE UPDATES ## # perform exponential moving average mean_dtype = theano.scalar.upcast(accept.dtype, avg_acceptance_rate.dtype) new_acceptance_rate = TT.add( avg_acceptance_slowness * avg_acceptance_rate, (1.0 - avg_acceptance_slowness) * accept.mean(dtype=mean_dtype)) - + # end-snippet-6 start-snippet-8 return [(positions, new_positions), (stepsize, new_stepsize), (avg_acceptance_rate, new_acceptance_rate)] - + # end-snippet-8 class HMC_sampler(object): """ diff --git a/code/logistic_cg.py b/code/logistic_cg.py index dba5d576..2540c038 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -80,10 +80,14 @@ def __init__(self, input, n_in, n_out): # initialize theta = (W,b) with 0s; W gets the shape (n_in, n_out), # while b is a vector of n_out elements, making theta a vector of # n_in*n_out + n_out elements - self.theta = theano.shared(value=numpy.zeros(n_in * n_out + n_out, - dtype=theano.config.floatX), - name='theta', - borrow=True) + self.theta = theano.shared( + value=numpy.zeros( + n_in * n_out + n_out, + dtype=theano.config.floatX + ), + name='theta', + borrow=True + ) # W is represented by the fisr n_in*n_out elements of theta self.W = self.theta[0:n_in * n_out].reshape((n_in, n_out)) # b is the rest (last n_out elements) @@ -123,8 +127,10 @@ def errors(self, y): # check if y has same dimension of y_pred if y.ndim != self.y_pred.ndim: - raise TypeError('y should have the same shape as self.y_pred', - ('y', target.type, 'y_pred', self.y_pred.type)) + raise TypeError( + 'y should have the same shape as self.y_pred', + ('y', target.type, 'y_pred', self.y_pred.type) + ) # check if y is of the correct datatype if y.dtype.startswith('int'): # the T.neq operator returns a vector of 0s and 1s, where 1 @@ -186,39 +192,48 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): # compile a theano function that computes the mistakes that are made by # the model on a minibatch - test_model = theano.function([minibatch_offset], classifier.errors(y), - givens={ - x: test_set_x[minibatch_offset:minibatch_offset + batch_size], - y: test_set_y[minibatch_offset:minibatch_offset + batch_size]}, - name="test") - - validate_model = theano.function([minibatch_offset], classifier.errors(y), - givens={ - x: valid_set_x[minibatch_offset: - minibatch_offset + batch_size], - y: valid_set_y[minibatch_offset: - minibatch_offset + batch_size]}, - name="validate") - - # compile a thenao function that returns the cost of a minibatch - batch_cost = theano.function([minibatch_offset], cost, - givens={ - x: train_set_x[minibatch_offset: - minibatch_offset + batch_size], - y: train_set_y[minibatch_offset: - minibatch_offset + batch_size]}, - name="batch_cost") + test_model = theano.function( + [minibatch_offset], + classifier.errors(y), + givens={ + x: test_set_x[minibatch_offset:minibatch_offset + batch_size], + y: test_set_y[minibatch_offset:minibatch_offset + batch_size] + }, + name="test" + ) + + validate_model = theano.function( + [minibatch_offset], + classifier.errors(y), + givens={ + x: valid_set_x[minibatch_offset: minibatch_offset + batch_size], + y: valid_set_y[minibatch_offset: minibatch_offset + batch_size] + }, + name="validate" + ) + + # compile a theano function that returns the cost of a minibatch + batch_cost = theano.function( + [minibatch_offset], + cost, + givens={ + x: train_set_x[minibatch_offset: minibatch_offset + batch_size], + y: train_set_y[minibatch_offset: minibatch_offset + batch_size] + }, + name="batch_cost" + ) # compile a theano function that returns the gradient of the minibatch # with respect to theta - batch_grad = theano.function([minibatch_offset], - T.grad(cost, classifier.theta), - givens={ - x: train_set_x[minibatch_offset: - minibatch_offset + batch_size], - y: train_set_y[minibatch_offset: - minibatch_offset + batch_size]}, - name="batch_grad") + batch_grad = theano.function( + [minibatch_offset], + T.grad(cost, classifier.theta), + givens={ + x: train_set_x[minibatch_offset: minibatch_offset + batch_size], + y: train_set_y[minibatch_offset: minibatch_offset + batch_size] + }, + name="batch_grad" + ) # creates a function that computes the average cost on the training set def train_fn(theta_value): @@ -265,16 +280,21 @@ def callback(theta_value): print ("Optimizing using scipy.optimize.fmin_cg...") start_time = time.clock() best_w_b = scipy.optimize.fmin_cg( - f=train_fn, - x0=numpy.zeros((n_in + 1) * n_out, dtype=x.dtype), - fprime=train_fn_grad, - callback=callback, - disp=0, - maxiter=n_epochs) + f=train_fn, + x0=numpy.zeros((n_in + 1) * n_out, dtype=x.dtype), + fprime=train_fn_grad, + callback=callback, + disp=0, + maxiter=n_epochs + ) end_time = time.clock() - print(('Optimization complete with best validation score of %f %%, with ' - 'test performance %f %%') % - (validation_scores[0] * 100., validation_scores[1] * 100.)) + print( + ( + 'Optimization complete with best validation score of %f %%, with ' + 'test performance %f %%' + ) + % (validation_scores[0] * 100., validation_scores[1] * 100.) + ) print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] + diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 62788b80..6904717e 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -72,7 +72,7 @@ def __init__(self, input, n_in, n_out): which the labels lie """ - + # start-snippet-1 # initialize with 0 the weights W as a matrix of shape (n_in, n_out) self.W = theano.shared( value=numpy.zeros( @@ -105,7 +105,8 @@ def __init__(self, input, n_in, n_out): # symbolic description of how to compute prediction as class whose # probability is maximal self.y_pred = T.argmax(self.p_y_given_x, axis=1) - + # end-snippet-1 + # parameters of the model self.params = [self.W, self.b] @@ -126,6 +127,7 @@ def negative_log_likelihood(self, y): Note: we use the mean instead of the sum so that the learning rate is less dependent on the batch size """ + # start-snippet-2 # y.shape[0] is (symbolically) the number of rows in y, i.e., # number of examples (call it n) in the minibatch # T.arange(y.shape[0]) is a symbolic vector which will contain @@ -137,6 +139,7 @@ def negative_log_likelihood(self, y): # the mean (across minibatch examples) of the elements in v, # i.e., the mean log-likelihood across the minibatch. return -T.mean(T.log(self.p_y_given_x)[T.arange(y.shape[0]), y]) + # end-snippet-2 def errors(self, y): """Return a float representing the number of errors in the minibatch @@ -150,8 +153,10 @@ def errors(self, y): # check if y has same dimension of y_pred if y.ndim != self.y_pred.ndim: - raise TypeError('y should have the same shape as self.y_pred', - ('y', target.type, 'y_pred', self.y_pred.type)) + raise TypeError( + 'y should have the same shape as self.y_pred', + ('y', target.type, 'y_pred', self.y_pred.type) + ) # check if y is of the correct datatype if y.dtype.startswith('int'): # the T.neq operator returns a vector of 0s and 1s, where 1 @@ -282,8 +287,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # generate symbolic variables for input (x and y represent a # minibatch) - x = T.matrix('x') # data, presented as rasterized images - y = T.ivector('y') # labels, presented as 1D vector of [int] labels + x = T.matrix('x') # data, presented as rasterized images + y = T.ivector('y') # labels, presented as 1D vector of [int] labels # construct the logistic regression class # Each MNIST image has size 28*28 @@ -317,6 +322,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, g_W = T.grad(cost=cost, wrt=classifier.W) g_b = T.grad(cost=cost, wrt=classifier.b) + # start-snippet-3 # specify how to update the parameters of the model as a list of # (variable, update expression) pairs. updates = [(classifier.W, classifier.W - learning_rate * g_W), @@ -334,6 +340,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) + # end-snippet-3 ############### # TRAIN MODEL # @@ -372,9 +379,15 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, for i in xrange(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) - print('epoch %i, minibatch %i/%i, validation error %f %%' % \ - (epoch, minibatch_index + 1, n_train_batches, - this_validation_loss * 100.)) + print( + 'epoch %i, minibatch %i/%i, validation error %f %%' % + ( + epoch, + minibatch_index + 1, + n_train_batches, + this_validation_loss * 100. + ) + ) # if we got the best validation score until now if this_validation_loss < best_validation_loss: @@ -390,19 +403,31 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, for i in xrange(n_test_batches)] test_score = numpy.mean(test_losses) - print((' epoch %i, minibatch %i/%i, test error of best' - ' model %f %%') % - (epoch, minibatch_index + 1, n_train_batches, - test_score * 100.)) + print( + ( + ' epoch %i, minibatch %i/%i, test error of best' + ' model %f %%' + ) % + ( + epoch, + minibatch_index + 1, + n_train_batches, + test_score * 100. + ) + ) if patience <= iter: done_looping = True break end_time = time.clock() - print(('Optimization complete with best validation score of %f %%,' - 'with test performance %f %%') % - (best_validation_loss * 100., test_score * 100.)) + print( + ( + 'Optimization complete with best validation score of %f %%,' + 'with test performance %f %%' + ) + % (best_validation_loss * 100., test_score * 100.) + ) print 'The code run for %d epochs, with %f epochs/sec' % ( epoch, 1. * epoch / (end_time - start_time)) print >> sys.stderr, ('The code for file ' + diff --git a/code/mlp.py b/code/mlp.py index 20e8c748..6f2fb92e 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -36,6 +36,7 @@ from logistic_sgd import LogisticRegression, load_data +# start-snippet-1 class HiddenLayer(object): def __init__(self, rng, input, n_in, n_out, W=None, b=None, activation=T.tanh): @@ -65,6 +66,7 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, layer """ self.input = input + # end-snippet-1 # `W` is initialized with `W_values` which is uniformely sampled # from sqrt(-6./(n_in+n_hidden)) and sqrt(6./(n_in+n_hidden)) @@ -108,6 +110,7 @@ def __init__(self, rng, input, n_in, n_out, W=None, b=None, self.params = [self.W, self.b] +# start-snippet-2 class MLP(object): """Multi-Layer Perceptron Class @@ -161,16 +164,20 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): n_in=n_hidden, n_out=n_out ) - + # end-snippet-2 start-snippet-3 # L1 norm ; one regularization option is to enforce L1 norm to # be small - self.L1 = abs(self.hiddenLayer.W).sum() \ - + abs(self.logRegressionLayer.W).sum() + self.L1 = ( + abs(self.hiddenLayer.W).sum() + + abs(self.logRegressionLayer.W).sum() + ) # square of L2 norm ; one regularization option is to enforce # square of L2 norm to be small - self.L2_sqr = (self.hiddenLayer.W ** 2).sum() \ - + (self.logRegressionLayer.W ** 2).sum() + self.L2_sqr = ( + (self.hiddenLayer.W ** 2).sum() + + (self.logRegressionLayer.W ** 2).sum() + ) # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the @@ -184,6 +191,7 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): # the parameters of the model are the parameters of the two layer it is # made out of self.params = self.hiddenLayer.params + self.logRegressionLayer.params + # end-snippet-3 def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, @@ -248,27 +256,38 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, n_out=10 ) + # start-snippet-4 # the cost we minimize during training is the negative log likelihood of # the model plus the regularization terms (L1 and L2); cost is expressed # here symbolically - cost = classifier.negative_log_likelihood(y) \ - + L1_reg * classifier.L1 \ - + L2_reg * classifier.L2_sqr + cost = ( + classifier.negative_log_likelihood(y) + + L1_reg * classifier.L1 + + L2_reg * classifier.L2_sqr + ) + # end-snippet-4 # compiling a Theano function that computes the mistakes that are made # by the model on a minibatch - test_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: test_set_x[index * batch_size:(index + 1) * batch_size], - y: test_set_y[index * batch_size:(index + 1) * batch_size]}) - - validate_model = theano.function(inputs=[index], - outputs=classifier.errors(y), - givens={ - x: valid_set_x[index * batch_size:(index + 1) * batch_size], - y: valid_set_y[index * batch_size:(index + 1) * batch_size]}) + test_model = theano.function( + inputs=[index], + outputs=classifier.errors(y), + givens={ + x: test_set_x[index * batch_size:(index + 1) * batch_size], + y: test_set_y[index * batch_size:(index + 1) * batch_size] + } + ) + + validate_model = theano.function( + inputs=[index], + outputs=classifier.errors(y), + givens={ + x: valid_set_x[index * batch_size:(index + 1) * batch_size], + y: valid_set_y[index * batch_size:(index + 1) * batch_size] + } + ) + # start-snippet-5 # compute the gradient of cost with respect to theta (sotred in params) # the resulting gradients will be stored in a list gparams gparams = [T.grad(cost, param) for param in classifier.params] @@ -297,6 +316,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, y: train_set_y[index * batch_size: (index + 1) * batch_size] } ) + # end-snippet-5 ############### # TRAIN MODEL # @@ -338,15 +358,23 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, in xrange(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) - print('epoch %i, minibatch %i/%i, validation error %f %%' % - (epoch, minibatch_index + 1, n_train_batches, - this_validation_loss * 100.)) + print( + 'epoch %i, minibatch %i/%i, validation error %f %%' % + ( + epoch, + minibatch_index + 1, + n_train_batches, + this_validation_loss * 100. + ) + ) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough - if this_validation_loss < best_validation_loss * \ - improvement_threshold: + if ( + this_validation_loss < best_validation_loss * + improvement_threshold + ): patience = max(patience, iter * patience_increase) best_validation_loss = this_validation_loss @@ -363,8 +391,8 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, test_score * 100.)) if patience <= iter: - done_looping = True - break + done_looping = True + break end_time = time.clock() print(('Optimization complete. Best validation score of %f %% ' diff --git a/code/rbm.py b/code/rbm.py index 064d7b37..875b8dd2 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -25,6 +25,7 @@ from logistic_sgd import load_data +# start-snippet-1 class RBM(object): """Restricted Boltzmann Machine (RBM) """ def __init__( @@ -123,6 +124,7 @@ def __init__( # **** WARNING: It is not a good idea to put things in this list # other than shared variables created in this function. self.params = [self.W, self.hbias, self.vbias] + # end-snippet-1 def free_energy(self, v_sample): ''' Function to compute the free energy ''' @@ -202,6 +204,7 @@ def gibbs_vhv(self, v0_sample): return [pre_sigmoid_h1, h1_mean, h1_sample, pre_sigmoid_v1, v1_mean, v1_sample] + # start-snippet-2 def get_cost_updates(self, lr=0.1, persistent=None, k=1): """This functions implements one step of CD-k or PCD-k @@ -230,36 +233,48 @@ def get_cost_updates(self, lr=0.1, persistent=None, k=1): chain_start = ph_sample else: chain_start = persistent - + # end-snippet-2 # perform actual negative phase # in order to implement CD-k/PCD-k we need to scan over the # function that implements one gibbs step k times. # Read Theano tutorial on scan for more information : # http://deeplearning.net/software/theano/library/scan.html # the scan will return the entire Gibbs chain - [pre_sigmoid_nvs, nv_means, nv_samples, - pre_sigmoid_nhs, nh_means, nh_samples], updates = \ - theano.scan(self.gibbs_hvh, - # the None are place holders, saying that - # chain_start is the initial state corresponding to the - # 6th output - outputs_info=[None, None, None, None, None, chain_start], - n_steps=k) - + ( + [ + pre_sigmoid_nvs, + nv_means, + nv_samples, + pre_sigmoid_nhs, + nh_means, + nh_samples + ], + updates + ) = theano.scan( + self.gibbs_hvh, + # the None are place holders, saying that + # chain_start is the initial state corresponding to the + # 6th output + outputs_info=[None, None, None, None, None, chain_start], + n_steps=k + ) + # start-snippet-3 # determine gradients on RBM parameters - # not that we only need the sample at the end of the chain + # note that we only need the sample at the end of the chain chain_end = nv_samples[-1] cost = T.mean(self.free_energy(self.input)) - T.mean( self.free_energy(chain_end)) # We must not compute the gradient through the gibbs sampling gparams = T.grad(cost, self.params, consider_constant=[chain_end]) - + # end-snippet-3 start-snippet-4 # constructs the update dictionary for gparam, param in zip(gparams, self.params): # make sure that the learning rate is of the right dtype - updates[param] = param - gparam * T.cast(lr, - dtype=theano.config.floatX) + updates[param] = param - gparam * T.cast( + lr, + dtype=theano.config.floatX + ) if persistent: # Note that this works only if persistent is a shared variable updates[persistent] = nh_samples[-1] @@ -271,6 +286,7 @@ def get_cost_updates(self, lr=0.1, persistent=None, k=1): pre_sigmoid_nvs[-1]) return monitoring_cost, updates + # end-snippet-4 def get_pseudo_likelihood_cost(self, updates): """Stochastic approximation to the pseudo-likelihood""" @@ -332,9 +348,12 @@ def get_reconstruction_cost(self, updates, pre_sigmoid_nv): """ cross_entropy = T.mean( - T.sum(self.input * T.log(T.nnet.sigmoid(pre_sigmoid_nv)) + + T.sum( + self.input * T.log(T.nnet.sigmoid(pre_sigmoid_nv)) + (1 - self.input) * T.log(1 - T.nnet.sigmoid(pre_sigmoid_nv)), - axis=1)) + axis=1 + ) + ) return cross_entropy @@ -397,13 +416,18 @@ def test_rbm(learning_rate=0.1, training_epochs=15, os.makedirs(output_folder) os.chdir(output_folder) + # start-snippet-5 # it is ok for a theano function to have no output # the purpose of train_rbm is solely to update the RBM parameters - train_rbm = theano.function([index], cost, - updates=updates, - givens={x: train_set_x[index * batch_size: - (index + 1) * batch_size]}, - name='train_rbm') + train_rbm = theano.function( + [index], + cost, + updates=updates, + givens={ + x: train_set_x[index * batch_size: (index + 1) * batch_size] + }, + name='train_rbm' + ) plotting_time = 0. start_time = time.clock() @@ -421,10 +445,14 @@ def test_rbm(learning_rate=0.1, training_epochs=15, # Plot filters after each training epoch plotting_start = time.clock() # Construct image from the weight matrix - image = Image.fromarray(tile_raster_images( - X=rbm.W.get_value(borrow=True).T, - img_shape=(28, 28), tile_shape=(10, 10), - tile_spacing=(1, 1))) + image = Image.fromarray( + tile_raster_images( + X=rbm.W.get_value(borrow=True).T, + img_shape=(28, 28), + tile_shape=(10, 10), + tile_spacing=(1, 1) + ) + ) image.save('filters_at_epoch_%i.png' % epoch) plotting_stop = time.clock() plotting_time += (plotting_stop - plotting_start) @@ -434,7 +462,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, pretraining_time = (end_time - start_time) - plotting_time print ('Training took %f minutes' % (pretraining_time / 60.)) - + # end-snippet-5 start-snippet-6 ################################# # Sampling from the RBM # ################################# @@ -443,20 +471,32 @@ def test_rbm(learning_rate=0.1, training_epochs=15, # pick random test examples, with which to initialize the persistent chain test_idx = rng.randint(number_of_test_samples - n_chains) - persistent_vis_chain = theano.shared(numpy.asarray( + persistent_vis_chain = theano.shared( + numpy.asarray( test_set_x.get_value(borrow=True)[test_idx:test_idx + n_chains], - dtype=theano.config.floatX)) - + dtype=theano.config.floatX + ) + ) + # end-snippet-6 start-snippet-7 plot_every = 1000 # define one step of Gibbs sampling (mf = mean-field) define a # function that does `plot_every` steps before returning the # sample for plotting - [presig_hids, hid_mfs, hid_samples, presig_vis, - vis_mfs, vis_samples], updates = \ - theano.scan(rbm.gibbs_vhv, - outputs_info=[None, None, None, None, - None, persistent_vis_chain], - n_steps=plot_every) + ( + [ + presig_hids, + hid_mfs, + hid_samples, + presig_vis, + vis_mfs, + vis_samples + ], + updates + ) = theano.scan( + rbm.gibbs_vhv, + outputs_info=[None, None, None, None, None, persistent_vis_chain], + n_steps=plot_every + ) # add to updates the shared variable that takes care of our persistent # chain :. @@ -464,28 +504,38 @@ def test_rbm(learning_rate=0.1, training_epochs=15, # construct the function that implements our persistent chain. # we generate the "mean field" activations for plotting and the actual # samples for reinitializing the state of our persistent chain - sample_fn = theano.function([], [vis_mfs[-1], vis_samples[-1]], - updates=updates, - name='sample_fn') + sample_fn = theano.function( + [], + [ + vis_mfs[-1], + vis_samples[-1] + ], + updates=updates, + name='sample_fn' + ) # create a space to store the image for plotting ( we need to leave # room for the tile_spacing as well) - image_data = numpy.zeros((29 * n_samples + 1, 29 * n_chains - 1), - dtype='uint8') + image_data = numpy.zeros( + (29 * n_samples + 1, 29 * n_chains - 1), + dtype='uint8' + ) for idx in xrange(n_samples): # generate `plot_every` intermediate samples that we discard, # because successive samples in the chain are too correlated vis_mf, vis_sample = sample_fn() print ' ... plotting sample ', idx image_data[29 * idx:29 * idx + 28, :] = tile_raster_images( - X=vis_mf, - img_shape=(28, 28), - tile_shape=(1, n_chains), - tile_spacing=(1, 1)) - # construct image + X=vis_mf, + img_shape=(28, 28), + tile_shape=(1, n_chains), + tile_spacing=(1, 1) + ) + # construct image image = Image.fromarray(image_data) image.save('samples.png') + # end-snippet-7 os.chdir('../') if __name__ == '__main__': diff --git a/code/rnnrbm.py b/code/rnnrbm.py index 42f04d8c..3b8ede88 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -207,16 +207,26 @@ def __init__( self.r = r self.dt = dt (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) = build_rnnrbm(r[1] - r[0], n_hidden, - n_hidden_recurrent) + updates_generate) = build_rnnrbm( + r[1] - r[0], + n_hidden, + n_hidden_recurrent + ) gradient = T.grad(cost, params, consider_constant=[v_sample]) - updates_train.update(((p, p - lr * g) for p, g in zip(params, - gradient))) - self.train_function = theano.function([v], monitor, - updates=updates_train) - self.generate_function = theano.function([], v_t, - updates=updates_generate) + updates_train.update( + ((p, p - lr * g) for p, g in zip(params, gradient)) + ) + self.train_function = theano.function( + [v], + monitor, + updates=updates_train + ) + self.generate_function = theano.function( + [], + v_t, + updates=updates_generate + ) def train(self, files, batch_size=100, num_epochs=200): '''Train the RNN-RBM via stochastic gradient descent (SGD) using MIDI diff --git a/code/utils.py b/code/utils.py index 6313ed2f..246c2569 100644 --- a/code/utils.py +++ b/code/utils.py @@ -64,8 +64,10 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), # tile_spacing[0] # out_shape[1] = (img_shape[1]+tile_spacing[1])*tile_shape[1] - # tile_spacing[1] - out_shape = [(ishp + tsp) * tshp - tsp for ishp, tshp, tsp - in zip(img_shape, tile_shape, tile_spacing)] + out_shape = [ + (ishp + tsp) * tshp - tsp + for ishp, tshp, tsp in zip(img_shape, tile_shape, tile_spacing) + ] if isinstance(X, tuple): assert len(X) == 4 @@ -90,8 +92,10 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), dt = out_array.dtype if output_pixel_vals: dt = 'uint8' - out_array[:, :, i] = numpy.zeros(out_shape, - dtype=dt) + channel_defaults[i] + out_array[:, :, i] = numpy.zeros( + out_shape, + dtype=dt + ) + channel_defaults[i] else: # use a recurrent call to compute the channel and store it # in the output diff --git a/doc/DBN.txt b/doc/DBN.txt index bc113fc0..f95dea50 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -143,8 +143,8 @@ much as possible the RBMs used to initialize the network and the MLP used for classification. .. literalinclude:: ../code/DBN.py - :start-after: class DBN(object): - :end-before: # The DBN is an MLP + :start-after: start-snippet-1 + :end-before: end-snippet-1 ``self.sigmoid_layers`` will store the feed-forward graphs which together form the MLP, while ``self.rbm_layers`` will store the RBMs used to pretrain each @@ -226,8 +226,8 @@ This function is applied to the training set for a fixed number of epochs given by ``pretraining_epochs``. .. literalinclude:: ../code/DBN.py - :start-after: PRETRAINING THE MODEL - :end-before: print >> sys.stderr + :start-after: start-snippet-2 + :end-before: end-snippet-2 The fine-tuning loop is very similar to the one in the :ref:`mlp` tutorial, the only difference being that we now use the functions given by diff --git a/doc/SdA.txt b/doc/SdA.txt index 91926bd9..8caa5c3f 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -74,8 +74,8 @@ the MLP share parameters, and the fact that autoencoders get as input latent representations of intermediate layers of the MLP. .. literalinclude:: ../code/SdA.py - :start-after: class SdA(object): - :end-before: # The SdA is an MLP + :start-after: start-snippet-1 + :end-before: end-snippet-1 ``self.sigmoid_layers`` will store the sigmoid layers of the MLP facade, while ``self.dA_layers`` will store the denoising autoencoder associated with the layers of the MLP. @@ -90,15 +90,15 @@ each denoising autoencoder such that they share the weight matrix and the bias of the encoding part with its corresponding sigmoid layer. .. literalinclude:: ../code/SdA.py - :start-after: [int] labels - :end-before: # We now need to add a logistic layer on top of the MLP + :start-after: start-snippet-2 + :end-before: end-snippet-2 All we need now is to add the logistic layer on top of the sigmoid layers such that we have an MLP. We will use the ``LogisticRegression`` class introduced in :ref:`logreg`. .. literalinclude:: ../code/SdA.py - :start-after: self.dA_layers.append(dA_layer) + :start-after: end-snippet-2 :end-before: def pretraining_functions The class also provides a method that generates training functions for @@ -144,8 +144,8 @@ The few lines of code below constructs the stacked denoising autoencoder : .. literalinclude:: ../code/SdA.py - :start-after: # numpy random generator - :end-before: PRETRAINING THE MODEL + :start-after: start-snippet-3 + :end-before: end-snippet-3 There are two stages in training this network, a layer-wise pre-training and fine-tuning afterwards. @@ -158,8 +158,8 @@ to the training set for a fixed number of epochs given by ``pretraining_epochs``. .. literalinclude:: ../code/SdA.py - :start-after: PRETRAINING THE MODEL - :end-before: FINETUNING THE MODEL + :start-after: start-snippet-4 + :end-before: end-snippet-4 The fine-tuning loop is very similar with the one in the :ref:`mlp`, the only difference is that we will use now the functions given by diff --git a/doc/dA.txt b/doc/dA.txt index 00902398..fcd5f452 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -106,7 +106,8 @@ autoencoder ( :math:`\mathbf{W}`, :math:`\mathbf{b}` and :math:`\mathbf{b'}`, since we are using tied weights in this tutorial ): .. literalinclude:: ../code/dA.py - :pyobject: dA.__init__ + :start-after: start-snippet-1 + :end-before: end-snippet-1 Note that we pass the symbolic ``input`` to the autoencoder as a parameter. This is such that later we can concatenate layers of diff --git a/doc/hmc.txt b/doc/hmc.txt index 989c823f..8348558b 100644 --- a/doc/hmc.txt +++ b/doc/hmc.txt @@ -195,17 +195,30 @@ computing `n\_steps` of HMC, using a given `stepsize`. The function prototype is as follows: .. literalinclude:: ../code/hmc/hmc.py - :pyobject: hmc_move + :start-after: start-snippet-1 + :end-before: end-snippet-1 We start by sampling random velocities, using the provided shared RandomStream object. Velocities are sampled independently for each dimension and for each particle under simulation, yielding a :math:`N \times D` matrix. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-2 + :end-before: end-snippet-2 + Since we now have an initial position and velocity, we can now call the `simulate\_dynamics` to obtain the proposal for the new state :math:`\chi'`. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-3 + :end-before: end-snippet-3 + We then accept/reject the proposed state based on the Metropolis algorithm. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-4 + :end-before: end-snippet-4 + where `metropolis\_hastings\_accept` and `hamiltonian` are helper functions, defined as follows. @@ -237,7 +250,8 @@ receives as parameters, a series of shared variables to update (`positions`, `st state. .. literalinclude:: ../code/hmc/hmc.py - :pyobject: hmc_updates + :start-after: start-snippet-5 + :end-before: end-snippet-5 Using the above code, the dictionary `{positions: new\_positions}` can be used to update the state of the sampler with either (1) the new state `final\_pos` @@ -255,6 +269,10 @@ average acceptance rate of the HMC move proposals (across many simulations), using an exponential moving average with time constant `1-avg\_acceptance\_slowness`. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-6 + :end-before: end-snippet-6 + If the average acceptance rate is larger than the `target\_acceptance\_rate`, we increase the `stepsize` by a factor of `stepsize\_inc` in order to increase the mixing rate of our chain. If the average acceptance rate is too low however, @@ -262,8 +280,16 @@ mixing rate of our chain. If the average acceptance rate is too low however, conservative mixing rate. The `clip`_ op allows us to maintain the `stepsize` in the range [`stepsize\_min`, `stepsize\_max`]. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-7 + :end-before: end-snippet-7 + The final updates list is then returned. +.. literalinclude:: ../code/hmc/hmc.py + :start-after: start-snippet-8 + :end-before: end-snippet-8 + **HMC_sampler** We finally tie everything together using the `HMC\_Sampler` class. Its main diff --git a/doc/lenet.txt b/doc/lenet.txt index d827c834..42809dca 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -411,8 +411,8 @@ the HiddenLayer class defined in :doc:`mlp` , we can instantiate the network as follows. .. literalinclude:: ../code/convolutional_mlp.py - :start-after: allocate symbolic variables for the data - :end-before: TRAIN MODEL + :start-after: start-snippet-1 + :end-before: end-snippet-1 We leave out the code that performs the actual training and early-stopping, since it is exactly the same as with an MLP. The interested reader can diff --git a/doc/logreg.txt b/doc/logreg.txt index ec6b4bfc..71180bc4 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -61,7 +61,8 @@ The code to do this in Theano is the following: .. literalinclude:: ../code/logistic_sgd.py - :pyobject: LogisticRegression.__init__ + :start-after: start-snippet-1 + :end-before: end-snippet-1 We first start by allocating symbolic variables for the inputs :math:`x,y`. Since the parameters of the model must maintain a persistent state throughout @@ -118,7 +119,8 @@ mini-batches (MSGD). See :ref:`opt_SGD` for more details. The following Theano code defines the (symbolic) loss for a given minibatch: .. literalinclude:: ../code/logistic_sgd.py - :pyobject: LogisticRegression.negative_log_likelihood + :start-after: start-snippet-2 + :end-before: end-snippet-2 .. note:: @@ -184,15 +186,15 @@ To get the gradients :math:`\partial{\ell}/\partial{W}` and .. literalinclude:: ../code/logistic_sgd.py :start-after: # compute the gradient of cost - :end-before: # specify how to update the parameters + :end-before: # start-snippet-3 ``g_W`` and ``g_b`` are again symbolic variables, which can be used as part of a computation graph. Performing one-step of gradient descent can then be done as follows: .. literalinclude:: ../code/logistic_sgd.py - :start-after: g_b = T.grad(cost=cost, wrt=classifier.b) - :end-before: # TRAIN MODEL + :start-after: start-snippet-3 + :end-before: end-snippet-3 The ``updates`` list contains, for each parameter, the stochastic gradient update operation. The ``givens`` dictionary indicates with diff --git a/doc/mlp.txt b/doc/mlp.txt index 7da0ffbd..ac6b7d6d 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -102,6 +102,10 @@ implementing a class that will represent any given hidden layer. To construct the MLP we will then only need to throw a logistic regression layer on top. +.. literalinclude:: ../code/mlp.py + :start-after: start-snippet-1 + :end-before: end-snippet-1 + The initial values for the weights of a hidden layer :math:`i` should be uniformly sampled from a symmetric interval that depends on the activation function. For :math:`tanh` activation function results obtained in [Xavier10]_ show that the @@ -116,7 +120,7 @@ both upward (activations flowing from inputs to outputs) and backward (gradients flowing from outputs to inputs). .. literalinclude:: ../code/mlp.py - :start-after: self.input = input + :start-after: end-snippet-1 :end-before: lin_output = T.dot(input, self.W) + self.b Note that we used a given non-linear function as the activation function of the hidden layer. By default this is ``tanh``, but in many cases we might want @@ -133,12 +137,17 @@ implemented in the previous tutorial :doc:`logreg`, you get the output of the MLP. You can see this in the following short implementation of the ``MLP`` class. +.. literalinclude:: ../code/mlp.py + :start-after: start-snippet-2 + :end-before: end-snippet-2 + In this tutorial we will also use L1 and L2 regularization (see :ref:`L1_L2_regularization`). For this, we need to compute the L1 norm and the squared L2 norm of the weights :math:`W^{(1)}, W^{(2)}`. .. literalinclude:: ../code/mlp.py - :pyobject: MLP + :start-after: start-snippet-3 + :end-before: end-snippet-3 As before, we train this model using stochastic gradient descent with mini-batches. The difference is that we modify the cost function to include the @@ -147,8 +156,8 @@ controlling the weight of these regularization terms in the total cost function. The code that computes the new cost is: .. literalinclude:: ../code/mlp.py - :start-after: # the cost we minimize during training - :end-before: # compiling a Theano function that computes the mistakes + :start-after: start-snippet-4 + :end-before: end-snippet-4 We then update the parameters of the model using the gradient. This code is almost identical to the one for logistic regression. Only the number of @@ -158,8 +167,8 @@ we created with the model ``params`` and parse it, computing a gradient at each step. .. literalinclude:: ../code/mlp.py - :start-after: # compute the gradient of cost - :end-before: # TRAIN MODEL + :start-after: start-snippet-5 + :end-before: end-snippet-5 Putting it All Together +++++++++++++++++++++++ diff --git a/doc/rbm.txt b/doc/rbm.txt index 2ccd80f3..b3880ae8 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -310,7 +310,8 @@ case the weight matrix and the hidden layer bias is shared with the corresponding sigmoidal layer of an MLP network. .. literalinclude:: ../code/rbm.py - :pyobject: RBM.__init__ + :start-after: start-snippet-1 + :end-before: end-snippet-1 Next step is to define functions which construct the symbolic graph associated with Eqs. :eq:`rbm_propup` - :eq:`rbm_propdown`. The code is as follows: @@ -378,8 +379,8 @@ We then add a ``get_cost_updates`` method, whose purpose is to generate the symb gradients for CD-k and PCD-k updates. .. literalinclude:: ../code/rbm.py - :start-after: def gibbs_vhv - :end-before: # perform actual negative phase + :start-after: start-snippet-2 + :end-before: end-snippet-2 Note that ``get_cost_updates`` takes as argument a variable called ``persistent``. This allows us to use the same code to implement both CD and PCD. To use PCD, ``persistent`` should refer to a shared variable which contains the @@ -392,8 +393,8 @@ Gibbs chain, sample that we need for getting the gradient (see Eq. :eq:`free_en op provided by Theano, therefore we urge the reader to look it up by following this `link `_. .. literalinclude:: ../code/rbm.py - :start-after: chain_start = persistent - :end-before: # determine gradients on RBM parameters + :start-after: end-snippet-2 + :end-before: start-snippet-3 Once we have the generated the chain we take the sample at the end of the chain to get the free energy of the negative phase. Note that the @@ -405,8 +406,8 @@ want (it will mess up our gradients) and therefore we need to indicate to ``consider_constant`` of ``T.grad``. .. literalinclude:: ../code/rbm.py - :start-after: # determine gradients on RBM parameters - :end-before: # constructs the update dictionary + :start-after: start-snippet-3 + :end-before: end-snippet-3 Finally, we add to the updates dictionary returned by scan (which contains updates rules for random states of ``theano_rng``) to contain the parameter @@ -414,8 +415,8 @@ updates. In the case of PCD, these should also update the shared variable containing the state of the Gibbs chain. .. literalinclude:: ../code/rbm.py - :start-after: # constructs the update dictionary - :end-before: def get_pseudo_likelihood_cost + :start-after: start-snippet-4 + :end-before: end-snippet-4 Tracking Progress ----------------- @@ -509,8 +510,8 @@ the filters after each training epoch. We train the RBM using PCD, as it has been shown to lead to a better generative model ([Tieleman08]_). .. literalinclude:: ../code/rbm.py - :start-after: os.chdir(output_folder) - :end-before: Sampling from the RBM + :start-after: start-snippet-5 + :end-before: end-snippet-5 Once the RBM is trained, we can then use the ``gibbs_vhv`` function to implement the Gibbs chain required for sampling. We initialize the Gibbs chain starting @@ -520,8 +521,8 @@ initialization. We again use Theano's ``scan`` op to do 1000 steps before each plotting. .. literalinclude:: ../code/rbm.py - :start-after: Sampling from the RBM - :end-before: plot_every = 1000 + :start-after: start-snippet-6 + :end-before: end-snippet-6 Next we create the 20 persistent chains in parallel to get our samples. To do so, we compile a theano function which performs one Gibbs step @@ -530,8 +531,8 @@ apply this function iteratively for a large number of steps, plotting the samples at every 1000 steps. .. literalinclude:: ../code/rbm.py - :start-after: find out the number of test - :end-before: os.chdir('../') + :start-after: start-snippet-7 + :end-before: end-snippet-7 Results +++++++ From 899adfdbd369119e6f7b9298f8673a96d6ed18ef Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Fri, 10 Oct 2014 18:29:23 -0700 Subject: [PATCH 056/417] Small adjustments to inclusion of code in ReStructuredText files. --- code/hmc/hmc.py | 24 +++++++++++++++--------- doc/DBN.txt | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index 740e3719..b7c292c0 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -321,15 +321,21 @@ def __init__(self, **kwargs): self.__dict__.update(kwargs) @classmethod - def new_from_shared_positions(cls, shared_positions, energy_fn, - initial_stepsize=0.01, target_acceptance_rate=.9, n_steps=20, - stepsize_dec=0.98, - stepsize_min=0.001, - stepsize_max=0.25, - stepsize_inc=1.02, - # used in geometric avg. 1.0 would be not moving at all - avg_acceptance_slowness=0.9, - seed=12345): + def new_from_shared_positions( + cls, + shared_positions, + energy_fn, + initial_stepsize=0.01, + target_acceptance_rate=.9, + n_steps=20, + stepsize_dec=0.98, + stepsize_min=0.001, + stepsize_max=0.25, + stepsize_inc=1.02, + # used in geometric avg. 1.0 would be not moving at all + avg_acceptance_slowness=0.9, + seed=12345 + ): """ :param shared_positions: theano ndarray shared var with many particle [initial] positions diff --git a/doc/DBN.txt b/doc/DBN.txt index f95dea50..bb0571eb 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -214,7 +214,7 @@ The few lines of code below constructs the deep belief network : .. literalinclude:: ../code/DBN.py :start-after: # numpy random generator - :end-before: ######################### + :end-before: start-snippet-2 There are two stages in training this network: (1) a layer-wise pre-training and (2) a fine-tuning stage. From 26d340273187162a7f08f4b921e6e8159e66e562 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Sat, 11 Oct 2014 06:20:48 -0700 Subject: [PATCH 057/417] Edits for clarification. --- doc/mlp.txt | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index ac6b7d6d..dd26072c 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -31,16 +31,17 @@ Multilayer Perceptron .. _GPU: http://deeplearning.net/software/theano/tutorial/using_gpu.html -The next architecture we are going to present using Theano is the single-hidden -layer Multi-Layer Perceptron (MLP). An MLP can be viewed as a logistic -regressor, where the input is first transformed using a learnt non-linear -transformation :math:`\Phi`. The purpose of this transformation is to project the +The next architecture we are going to present using Theano is the +single-hidden-layer Multi-Layer Perceptron (MLP). An MLP can be viewed as a +logistic regression classifier where the input is first transformed using a +learnt non-linear transformation :math:`\Phi`. This transformation projects the input data into a space where it becomes linearly separable. This intermediate -layer is referred to as a **hidden layer**. A single hidden layer is -sufficient to make MLPs a **universal approximator**. However we will see later -on that there are substantial benefits to using many such hidden layers, i.e. the -very premise of **deep learning**. See these course notes for an `introduction -to MLPs, the back-propagation algorithm, and how to train MLPs `_. +layer is referred to as a **hidden layer**. A single hidden layer is sufficient +to make MLPs a **universal approximator**. However we will see later on that +there are substantial benefits to using many such hidden layers, i.e. the very +premise of **deep learning**. See these course notes for an `introduction to +MLPs, the back-propagation algorithm, and how to train MLPs +`_. This tutorial will again tackle the problem of MNIST digit classification. @@ -54,10 +55,9 @@ follows: .. figure:: images/mlp.png :align: center -Formally, a one-hidden layer MLP constitutes a function :math:`f: R^D \rightarrow R^L`, -where :math:`D` is the size of input vector :math:`x` -and :math:`L` is the size of the output vector :math:`f(x)`, such that, -in matrix notation: +Formally, a one-hidden-layer MLP computes a function :math:`f: R^D \rightarrow +R^L`, where :math:`D` is the size of input vector :math:`x` and :math:`L` is +the size of the output vector :math:`f(x)`, such that, in matrix notation: .. math:: @@ -97,8 +97,8 @@ cover this in the tutorial ! Going from logistic regression to MLP +++++++++++++++++++++++++++++++++++++ -This tutorial will focus on a single-layer MLP. We start off by -implementing a class that will represent any given hidden layer. To +This tutorial will focus on a single-hidden-layer MLP. We start off by +implementing a class that will represent a hidden layer. To construct the MLP we will then only need to throw a logistic regression layer on top. @@ -132,7 +132,7 @@ to use something else. If you look into theory this class implements the graph that computes the hidden layer value :math:`h(x) = \Phi(x) = s(b^{(1)} + W^{(1)} x)`. -If you give this as input to the ``LogisticRegression`` class, +If you give this value as input to the ``LogisticRegression`` class, implemented in the previous tutorial :doc:`logreg`, you get the output of the MLP. You can see this in the following short implementation of the ``MLP`` class. From efe2eaf839dabb2d9664a04142b84cad9391500d Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Wed, 15 Oct 2014 12:44:52 -0700 Subject: [PATCH 058/417] Edits suggested by @lamblin. --- doc/mlp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index dd26072c..4ee9fdd7 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -55,7 +55,7 @@ follows: .. figure:: images/mlp.png :align: center -Formally, a one-hidden-layer MLP computes a function :math:`f: R^D \rightarrow +Formally, a one-hidden-layer MLP is a function :math:`f: R^D \rightarrow R^L`, where :math:`D` is the size of input vector :math:`x` and :math:`L` is the size of the output vector :math:`f(x)`, such that, in matrix notation: @@ -132,7 +132,7 @@ to use something else. If you look into theory this class implements the graph that computes the hidden layer value :math:`h(x) = \Phi(x) = s(b^{(1)} + W^{(1)} x)`. -If you give this value as input to the ``LogisticRegression`` class, +If you give this graph as input to the ``LogisticRegression`` class, implemented in the previous tutorial :doc:`logreg`, you get the output of the MLP. You can see this in the following short implementation of the ``MLP`` class. From 42ce957964b769ea5410078225b8ecd81a104c94 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Sat, 11 Oct 2014 09:57:23 -0700 Subject: [PATCH 059/417] Edits for clarification. --- doc/lenet.txt | 205 +++++++++++++++++++++++++------------------------- 1 file changed, 103 insertions(+), 102 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 42809dca..b62f921a 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -52,87 +52,87 @@ Convolutional Neural Networks (LeNet) Motivation ++++++++++ -Convolutional Neural Networks (CNN) are variants of MLPs which are inspired from -biology. From Hubel and Wiesel's early work on the cat's visual cortex [Hubel68]_, -we know there exists a complex arrangement of cells within the visual cortex. -These cells are sensitive to small sub-regions of the input space, called a -**receptive field**, and are tiled in such a way as to cover the entire visual -field. These filters are local in input space and are thus better suited to -exploit the strong spatially local correlation present in natural images. - -Additionally, two basic cell types have been identified: simple cells (S) and -complex cells (C). Simple cells (S) respond maximally to specific edge-like -stimulus patterns within their receptive field. Complex cells (C) have larger -receptive fields and are locally invariant to the exact position of the -stimulus. - -The visual cortex being the most powerful "vision" system in existence, it -seems natural to emulate its behavior. Many such neurally inspired models can be -found in the litterature. To name a few: the NeoCognitron [Fukushima]_, HMAX -[Serre07]_ and LeNet-5 [LeCun98]_, which will be the focus of this tutorial. +Convolutional Neural Networks (CNN) are biologically-inspired variants of MLPs. +From Hubel and Wiesel's early work on the cat's visual cortex [Hubel68]_, we +know the visual cortex contains a complex arrangement of cells. These cells are +sensitive to small sub-regions of the visual field, called a *receptive +field*. The sub-regions are tiled to cover the entire visual field. These +cells act as local filters over the input space and are well-suited to exploit +the strong spatially local correlation present in natural images. + +Additionally, two basic cell types have been identified: Simple cells respond +maximally to specific edge-like patterns within their receptive field. Complex +cells have larger receptive fields and are locally invariant to the exact +position of the pattern. + +The animal visual cortex being the most powerful visual processing system in +existence, it seems natural to emulate its behavior. Hence, many +neurally-inspired models can be found in the literature. To name a few: the +NeoCognitron [Fukushima]_, HMAX [Serre07]_ and LeNet-5 [LeCun98]_, which will +be the focus of this tutorial. Sparse Connectivity +++++++++++++++++++ -CNNs exploit spatially local correlation by enforcing a local connectivity pattern between -neurons of adjacent layers. The input hidden units in the m-th layer are -connected to a local subset of units in the (m-1)-th layer, which have spatially -contiguous receptive fields. We can illustrate this graphically as follows: +CNNs exploit spatially-local correlation by enforcing a local connectivity +pattern between neurons of adjacent layers. In other words, the inputs of +hidden units in layer **m** are from a subset of units in layer **m-1**, units +that have spatially contiguous receptive fields. We can illustrate this +graphically as follows: .. figure:: images/sparse_1D_nn.png :align: center -Imagine that layer **m-1** is the input retina. -In the above, units in layer **m** -have receptive fields of width 3 with respect to the input retina and are thus only -connected to 3 adjacent neurons in the layer below (the retina). -Units in layer **m** have -a similar connectivity with the layer below. We say that their receptive -field with respect to the layer below is also 3, but their receptive field -with respect to the input is larger (it is 5). -The architecture thus -confines the learnt "filters" (corresponding to the input producing the strongest response) to be a spatially local pattern -(since each unit is unresponsive to variations outside of its receptive field with respect to the retina). -As shown above, stacking many such -layers leads to "filters" (not anymore linear) which become increasingly "global" however (i.e -spanning a larger region of pixel space). For example, the unit in hidden -layer **m+1** can encode a non-linear feature of width 5 (in terms of pixel -space). +Imagine that layer **m-1** is the input retina. In the above figure, units in +layer **m** have receptive fields of width 3 in the input retina and are thus +only connected to 3 adjacent neurons in the retina layer. Units in layer **m** +have a similar connectivity with the layer below. We say that their receptive +field with respect to the layer below is also 3, but their receptive field with +respect to the input is larger (5). Each unit is unresponsive to variations +outside of its receptive field with respect to the retina. The architecture +thus ensures that the learnt "filters" produce the strongest response to a +spatially local input pattern. + +However, as shown above, stacking many such layers leads to (non-linear) +"filters" that become increasingly "global" (i.e. responsive to a larger region +of pixel space). For example, the unit in hidden layer **m+1** can encode a +non-linear feature of width 5 (in terms of pixel space). Shared Weights ++++++++++++++ -In CNNs, each sparse filter :math:`h_i` is additionally replicated across the -entire visual field. These "replicated" units form a **feature map**, which -share the same parametrization, i.e. the same weight vector and the same bias. +In addition, in CNNs, each filter :math:`h_i` is replicated across the entire +visual field. These replicated units share the same parameterization (weight +vector and bias) and form a *feature map*. .. figure:: images/conv_1D_nn.png :align: center In the above figure, we show 3 hidden units belonging to the same feature map. -Weights of the same color are shared, i.e. are constrained to be identical. -Gradient descent can still be used to learn such shared parameters, and -requires only a small change to the original algorithm. The gradient of a -shared weight is simply the sum of the gradients of the parameters being -shared. +Weights of the same color are shared---constrained to be identical. Gradient +descent can still be used to learn such shared parameters, with only a small +change to the original algorithm. The gradient of a shared weight is simply the +sum of the gradients of the parameters being shared. -Why are shared weights interesting ? Replicating units in this way allows for -features to be detected regardless of their position in the visual field. -Additionally, weight sharing offers a very efficient way to do this, since it -greatly reduces the number of free parameters to learn. By controlling model -capacity, CNNs tend to achieve better generalization on vision problems. +Replicating units in this way allows for features to be detected *regardless +of their position in the visual field.* Additionally, weight sharing increases +learning efficiency by greatly reducing the number of free parameters being +learnt. The constraints on the model enable CNNs to achieve better +generalization on vision problems. Details and Notation ++++++++++++++++++++ -Conceptually, a feature map is obtained by convolving the input image with a -linear filter, adding a bias term and then applying a non-linear function. If -we denote the k-th feature map at a given layer as :math:`h^k`, whose filters -are determined by the weights :math:`W^k` and bias :math:`b_k`, then the -feature map :math:`h^k` is obtained as follows (for :math:`tanh` non-linearities): +A feature map is obtained by repeated application of a function across +sub-regions of the entire image, in other words, by *convolution* of the +input image with a linear filter, adding a bias term and then applying a +non-linear function. If we denote the k-th feature map at a given layer as +:math:`h^k`, whose filters are determined by the weights :math:`W^k` and bias +:math:`b_k`, then the feature map :math:`h^k` is obtained as follows (for +:math:`tanh` non-linearities): .. math:: h^k_{ij} = \tanh ( (W^k * x)_{ij} + b_k ). @@ -144,40 +144,40 @@ feature map :math:`h^k` is obtained as follows (for :math:`tanh` non-linearities This can be extended to 2D as follows: :math:`o[m,n] = f[m,n]*g[m,n] = \sum_{u=-\infty}^{\infty} \sum_{v=-\infty}^{\infty} f[u,v] g[m-u,n-v]`. -To form a richer representation of the data, hidden layers are composed of -a set of multiple feature maps, :math:`\{h^{(k)}, k=0..K\}`. -The weights :math:`W` of this layer can be parametrized as a 4D tensor -(destination feature map index, source feature map index, source vertical position index, source horizontal position index) -and -the biases :math:`b` as a vector (one element per destination feature map index). -We illustrate this graphically as follows: +To form a richer representation of the data, each hidden layer is composed of +*multiple* feature maps, :math:`\{h^{(k)}, k=0..K\}`. The weights :math:`W` of +a hidden layer can be represented in a 4D tensor containing elements for every +combination of destination feature map, source feature map, source vertical +position, and source horizontal position. The biases :math:`b` can be +represented as a vector containing one element for every destination feature +map. We illustrate this graphically as follows: .. figure:: images/cnn_explained.png :align: center **Figure 1**: example of a convolutional layer -Here, we show two layers of a CNN, containing 4 feature maps at layer (m-1) -and 2 feature maps (:math:`h^0` and :math:`h^1`) at layer m. Pixels (neuron outputs) in -:math:`h^0` and :math:`h^1` (outlined as blue and red squares) are computed -from pixels of layer (m-1) which fall within their 2x2 receptive field in the -layer below (shown -as colored rectangles). Notice how the receptive field spans all four input -feature maps. The weights :math:`W^0` and :math:`W^1` of :math:`h^0` and -:math:`h^1` are thus 3D weight tensors. The leading dimension indexes the -input feature maps, while the other two refer to the pixel coordinates. +The figure shows two layers of a CNN. **Layer m-1** contains four feature maps. +**Hidden layer m** contains two feature maps (:math:`h^0` and :math:`h^1`). +Pixels (neuron outputs) in :math:`h^0` and :math:`h^1` (outlined as blue and +red squares) are computed from pixels of layer (m-1) which fall within their +2x2 receptive field in the layer below (shown as colored rectangles). Notice +how the receptive field spans all four input feature maps. The weights +:math:`W^0` and :math:`W^1` of :math:`h^0` and :math:`h^1` are thus 3D weight +tensors. The leading dimension indexes the input feature maps, while the other +two refer to the pixel coordinates. Putting it all together, :math:`W^{kl}_{ij}` denotes the weight connecting each pixel of the k-th feature map at layer m, with the pixel at coordinates (i,j) of the l-th feature map of layer (m-1). -The ConvOp -++++++++++ +The Convolution Operator +++++++++++++++++++++++++ -ConvOp is the main workhorse for implementing a convolutional layer in Theano. -It is meant to replicate the behaviour of scipy.signal.convolve2d. Conceptually, -the ConvOp (once instantiated) takes two symbolic inputs: +The ``conv2d`` function in Theano is the main workhorse for implementing a +convolutional layer. It replicates the behaviour of scipy.signal.convolve2d. +``conv2d`` takes two symbolic inputs: * a 4D tensor corresponding to a mini-batch of input images. The shape of the @@ -284,38 +284,39 @@ This should generate the following output. Notice that a randomly initialized filter acts very much like an edge detector! -Also of note, remark that we use the same weight initialization formula as -with the MLP. Weights are sampled randomly from a uniform distribution in the -range [-1/fan-in, 1/fan-in], where fan-in is the number of inputs to a hidden -unit. For MLPs, this was the number of units in the layer below. For CNNs -however, we have to take into account the number of input feature maps and the -size of the receptive fields. +Note that we use the same weight initialization formula as with the MLP. +Weights are sampled randomly from a uniform distribution in the range +[-1/fan-in, 1/fan-in], where fan-in is the number of inputs to a hidden unit. +For MLPs, this was the number of units in the layer below. For CNNs however, we +have to take into account the number of input feature maps and the size of the +receptive fields. MaxPooling ++++++++++ -Another important concept of CNNs is that of max-pooling, which is a form of +Another important concept of CNNs is *max-pooling,* which is a form of non-linear down-sampling. Max-pooling partitions the input image into a set of non-overlapping rectangles and, for each such sub-region, outputs the maximum value. -Max-pooling is useful in vision for two reasons: (1) it reduces the -computational complexity for upper layers and (2) it provides a form of -translation invariance. To understand the invariance argument, imagine -cascading a max-pooling layer with a convolutional layer. There are 8 -directions in which one can translate the input image by a single pixel. If -max-pooling is done over a 2x2 region, 3 out of these 8 possible -configurations will produce exactly the same output at the convolutional -layer. For max-pooling over a 3x3 window, this jumps to 5/8. +Max-pooling is useful in vision for two reasons: + #. By eliminating non-maximal values, it reduces computation for upper layers + + #. It provides a form of translation invariance. Imagine + cascading a max-pooling layer with a convolutional layer. There are 8 + directions in which one can translate the input image by a single pixel. + If max-pooling is done over a 2x2 region, 3 out of these 8 possible + configurations will produce exactly the same output at the convolutional + layer. For max-pooling over a 3x3 window, this jumps to 5/8. -Since it provides additional robustness to position, max-pooling is thus a -"smart" way of reducing the dimensionality of intermediate representations. + Since it provides additional robustness to position, max-pooling is thus a + "smart" way of reducing the dimensionality of intermediate representations. -Max-pooling is done in Theano by way of ``theano.tensor.signal.downsample.max_pool_2d``. -This function takes as input an N dimensional tensor (with N >= 2), a -downscaling factor and performs max-pooling over the 2 trailing dimensions of -the tensor. +Max-pooling is done in Theano by way of +``theano.tensor.signal.downsample.max_pool_2d``. This function takes as input +an N dimensional tensor (where N >= 2) and a downscaling factor and performs +max-pooling over the 2 trailing dimensions of the tensor. An example is worth a thousand words: @@ -366,10 +367,10 @@ This should generate the following output: [ 0.66379465 0.94459476 0.58655504] [ 0.90340192 0.80739129 0.39767684]] -Note that contrary to most Theano code, the ``max_pool_2d`` operation is a little -*special*. It requires the downscaling factor ``ds`` (tuple of length 2 containing -downscaling factors for image width and height) to be known at graph build -time. This may change in the near future. +Note that compared to most Theano code, the ``max_pool_2d`` operation is a +little *special*. It requires the downscaling factor ``ds`` (tuple of length 2 +containing downscaling factors for image width and height) to be known at graph +build time. This may change in the near future. The Full Model: LeNet From bef5b511daa00b2864377c89de25b2cee27f590c Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Wed, 15 Oct 2014 12:57:51 -0700 Subject: [PATCH 060/417] Additional minor edits. --- doc/lenet.txt | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index b62f921a..dbf9ac28 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -10,16 +10,18 @@ Convolutional Neural Networks (LeNet) `floatX`_, `downsample`_ , `conv2d`_, `dimshuffle`_. If you intend to run the code on GPU also read `GPU`_. - To run this example on a GPU, you need a good GPU. First, it need - at least 1G of GPU RAM and possibly more if your monitor is + To run this example on a GPU, you need a good GPU. It needs + at least 1GB of GPU RAM. More may be required if your monitor is connected to the GPU. - Second, when the GPU is connected to the monitor, there is a limit + When the GPU is connected to the monitor, there is a limit of a few seconds for each GPU function call. This is needed as - current GPU can't be used for the monitor while doing - computation. If there wasn't this limit, the screen would freeze - for too long and this look as if the computer froze. User don't - like this. This example hit this limit with medium GPU. When the + current GPUs can't be used for the monitor while doing + computation. Without this limit, the screen would freeze + for too long and make it look as if the computer froze. + This example hits this limit with medium-quality GPUs. + + When the GPU isn't connected to a monitor, there is no time limit. You can lower the batch size to fix the time out problem. @@ -301,7 +303,7 @@ a set of non-overlapping rectangles and, for each such sub-region, outputs the maximum value. Max-pooling is useful in vision for two reasons: - #. By eliminating non-maximal values, it reduces computation for upper layers + #. By eliminating non-maximal values, it reduces computation for upper layers. #. It provides a form of translation invariance. Imagine cascading a max-pooling layer with a convolutional layer. There are 8 @@ -310,7 +312,7 @@ Max-pooling is useful in vision for two reasons: configurations will produce exactly the same output at the convolutional layer. For max-pooling over a 3x3 window, this jumps to 5/8. - Since it provides additional robustness to position, max-pooling is thus a + Since it provides additional robustness to position, max-pooling is a "smart" way of reducing the dimensionality of intermediate representations. Max-pooling is done in Theano by way of From 988252606e9d9ae23cbcc77e40be4f8d3e961569 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Wed, 15 Oct 2014 16:43:55 -0700 Subject: [PATCH 061/417] Edits suggested by @lamblin. --- doc/lenet.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index dbf9ac28..7971cc7e 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -13,15 +13,13 @@ Convolutional Neural Networks (LeNet) To run this example on a GPU, you need a good GPU. It needs at least 1GB of GPU RAM. More may be required if your monitor is connected to the GPU. - + When the GPU is connected to the monitor, there is a limit of a few seconds for each GPU function call. This is needed as current GPUs can't be used for the monitor while doing computation. Without this limit, the screen would freeze for too long and make it look as if the computer froze. - This example hits this limit with medium-quality GPUs. - - When the + This example hits this limit with medium-quality GPUs. When the GPU isn't connected to a monitor, there is no time limit. You can lower the batch size to fix the time out problem. @@ -177,9 +175,8 @@ each pixel of the k-th feature map at layer m, with the pixel at coordinates The Convolution Operator ++++++++++++++++++++++++ -The ``conv2d`` function in Theano is the main workhorse for implementing a -convolutional layer. It replicates the behaviour of scipy.signal.convolve2d. -``conv2d`` takes two symbolic inputs: +ConvOp is the main workhorse for implementing a convolutional layer in Theano. +ConvOp is used by ``theano.tensor.signal.conv2d``, which takes two symbolic inputs: * a 4D tensor corresponding to a mini-batch of input images. The shape of the From 0150aad5cc0523d13b421e7194f47873d6bf40eb Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Fri, 10 Oct 2014 23:27:36 -0700 Subject: [PATCH 062/417] Edits for clarification. Edits suggested by @lamblin. --- doc/logreg.txt | 124 ++++++++++++++++++++++--------------------------- 1 file changed, 56 insertions(+), 68 deletions(-) diff --git a/doc/logreg.txt b/doc/logreg.txt index 71180bc4..ab27f95b 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -39,55 +39,40 @@ The Model Logistic regression is a probabilistic, linear classifier. It is parametrized by a weight matrix :math:`W` and a bias vector :math:`b`. Classification is -done by projecting data points onto a set of hyperplanes, the distance to -which reflects a class membership probability. +done by projecting an input vector onto a set of hyperplanes, each of which +corresponds to a class. The distance from the input to a hyperplane reflects +the probability that the input is a member of the corresponding class. -Mathematically, this can be written as: +Mathematically, the probability that an input vector :math:`x` is a member of a +class :math:`i`, a value of a stochastic variable :math:`Y`, can be written as: .. math:: P(Y=i|x, W,b) &= softmax_i(W x + b) \\ &= \frac {e^{W_i x + b_i}} {\sum_j e^{W_j x + b_j}} -The output of the model or prediction is then done by taking the argmax of the vector whose i'th element is P(Y=i|x). +The model's prediction :math:`y_{pred}` is the class whose probability is maximal, specifically: .. math:: y_{pred} = {\rm argmax}_i P(Y=i|x,W,b) The code to do this in Theano is the following: -.. literalinclude:: ../code/logistic_sgd.py - :start-after: index = T.lscalar() - :end-before: # construct the logistic regression class - - .. literalinclude:: ../code/logistic_sgd.py :start-after: start-snippet-1 :end-before: end-snippet-1 -We first start by allocating symbolic variables for the inputs :math:`x,y`. Since the parameters of the model must maintain a persistent state throughout -training, we allocate shared variables for :math:`W,b`. -This declares them both as being symbolic Theano variables, but also -initializes their contents. The dot and softmax operators are then used to compute the vector -:math:`P(Y|x, W,b)`. The resulting variable p_y_given_x is a symbolic variable -of vector-type. - -Up to this point, we have only defined the graph of computations which Theano -should perform. To get the actual numerical value of :math:`P(Y|x, W,b)`, we -must create a function ``get_p_y_given_x``, which takes as input ``x`` and -returns ``p_y_given_x``. We can then index its return value with the -index :math:`i` to get the membership probability of the :math:`i` th class. - -Now let's finish building the Theano graph. To get the actual model -prediction, we can use the ``T.argmax`` operator, which will return the index at -which ``p_y_given_x`` is maximal (i.e. the class with maximum probability). - -Again, to calculate the actual prediction for a given input, we construct a -function ``classify``. This function takes as argument a batch of inputs x (as a matrix), -and outputs a vector containing the predicted class for each example (row) in x. - -Now of course, the model we have defined so far does not do anything useful yet, -since its parameters are still in their initial random state. The following +training, we allocate shared variables for :math:`W,b`. This declares them both +as being symbolic Theano variables, but also initializes their contents. The +dot and softmax operators are then used to compute the vector :math:`P(Y|x, +W,b)`. The result ``p_y_given_x`` is a symbolic variable of vector-type. + +To get the actual model prediction, we can use the ``T.argmax`` operator, which +will return the index at which ``p_y_given_x`` is maximal (i.e. the class with +maximum probability). + +Now of course, the model we have defined so far does not do anything useful +yet, since its parameters are still in their initial state. The following section will thus cover how to learn the optimal parameters. @@ -146,25 +131,25 @@ We instantiate this class as follows: :start-after: index = T.lscalar() :end-before: # the cost we minimize during -Note that the inputs x and y are defined outside the scope of the -``LogisticRegression`` object. Since the class requires the input x to build its -graph however, it is passed as a parameter of the ``__init__`` function. -This is usefull in the case when you would want to concatenate such -classes to form a deep network (case in which the input is not a new -variable but the output of the layer below). While in this example we -will not do that, the tutorials are designed such that the code is as -similar as possible among them, making it easy to go from one tutorial -to the other. +We start by allocating symbolic variables for the training inputs :math:`x` and +their corresponding classes :math:`y`. Note that ``x`` and ``y`` are defined +outside the scope of the ``LogisticRegression`` object. Since the class +requires the input to build its graph, it is passed as a parameter of the +``__init__`` function. This is useful in case you want to connect instances of +such classes to form a deep network. The output of one layer can be passed as +the input of the layer above. (This tutorial does not build a multi-layer +network, but this code will be reused in future tutorials that do.) -The last step involves defining a (symbolic) cost variable to minimize, using -the instance method ``classifier.negative_log_likelihood``. +Finally, we define a (symbolic) ``cost`` variable to minimize, using the instance +method ``classifier.negative_log_likelihood``. .. literalinclude:: ../code/logistic_sgd.py :start-after: classifier = LogisticRegression(input=x, n_in=28 * 28, n_out=10) :end-before: # compiling a Theano function that computes the mistakes -Note how x is an implicit symbolic input to the symbolic definition of cost, -here, because classifier.__init__ has defined its symbolic variables in terms of x. +Note that ``x`` is an implicit symbolic input to the definition of ``cost``, +because the symbolic variables of ``classifier`` were defined in terms of ``x`` +at initialization. Learning the Model ++++++++++++++++++ @@ -177,7 +162,7 @@ models, as expressions for :math:`\partial{\ell}/\partial{\theta}` can get fairly complex, especially when taking into account problems of numerical stability. -With Theano, this work is greatly simplified as it performs +With Theano, this work is greatly simplified. It performs automatic differentiation and applies certain math transforms to improve numerical stability. @@ -188,33 +173,35 @@ To get the gradients :math:`\partial{\ell}/\partial{W}` and :start-after: # compute the gradient of cost :end-before: # start-snippet-3 -``g_W`` and ``g_b`` are again symbolic variables, which can be used as part of a -computation graph. Performing one-step of gradient descent can then be done as -follows: +``g_W`` and ``g_b`` are symbolic variables, which can be used as part +of a computation graph. The function ``train_model``, which performs one step +of gradient descent, can then be defined as follows: .. literalinclude:: ../code/logistic_sgd.py :start-after: start-snippet-3 :end-before: end-snippet-3 -The ``updates`` list contains, for each parameter, the -stochastic gradient update operation. The ``givens`` dictionary indicates with -what to replace certain variables of the graph. The function ``train_model`` is then -defined such that: +``updates`` is a list of pairs. In each pair, the first element is the symbolic +variable to be updated in the step, and the second element is the symbolic +function for calculating its new value. Similarly, ``givens`` is a dictionary +whose keys are symbolic variables and whose values specify +their replacements during the step. The function ``train_model`` is then defined such +that: -* the input is the mini-batch index ``index`` that together with the batch - size( which is not an input since it is fixed) defines :math:`x` with +* the input is the mini-batch index ``index`` that, together with the batch + size (which is not an input since it is fixed) defines :math:`x` with corresponding labels :math:`y` * the return value is the cost/loss associated with the x, y defined by the ``index`` -* on every function call, it will first replace ``x`` and ``y`` with the - corresponding slices from the training set as defined by the - ``index`` and afterwards it will evaluate the cost +* on every function call, it will first replace ``x`` and ``y`` with the slices + from the training set specified by ``index``. Then, it will evaluate the cost associated with that minibatch and apply the operations defined by the ``updates`` list. -Each time ``train_model(index)`` function is called, it will thus compute and -return the appropriate cost, while also performing a step of MSGD. The entire -learning algorithm thus consists in looping over all examples in the dataset, +Each time ``train_model(index)`` is called, it will thus compute and return the +cost of a minibatch, while also performing a step of MSGD. The entire learning +algorithm thus consists in looping over all examples in the dataset, considering +all the examples in one minibatch at a time, and repeatedly calling the ``train_model`` function. @@ -232,13 +219,14 @@ The code is as follows: .. literalinclude:: ../code/logistic_sgd.py :pyobject: LogisticRegression.errors -We then create a function ``test_model`` and a function ``validate_model``, which we can call to retrieve this -value. As you will see shortly, ``validate_model`` is key to our early-stopping -implementation (see :ref:`opt_early_stopping`). Both of these function -will get as input a batch offset and will compute the number of -missclassified examples for that mini-batch. The only difference between them -is that one draws its batches from the testing set, while -the other from the validation set. +We then create a function ``test_model`` and a function ``validate_model``, +which we can call to retrieve this value. As you will see shortly, +``validate_model`` is key to our early-stopping implementation (see +:ref:`opt_early_stopping`). These functions take a minibatch index and compute, +for the examples in that minibatch, the number that were misclassified by the +model. The only difference between them is that ``test_model`` draws its +minibatches from the testing set, while ``validate_model`` draws its from the +validation set. .. literalinclude:: ../code/logistic_sgd.py :start-after: cost = classifier.negative_log_likelihood(y) From 195e9fe42d91b5992c89cd312cef5aa10c9e44b3 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Sat, 11 Oct 2014 12:16:48 -0700 Subject: [PATCH 063/417] Edits to denoising autoencoder tutorial for clarity. --- doc/dA.txt | 231 ++++++++++++++++++++++++++--------------------------- 1 file changed, 113 insertions(+), 118 deletions(-) diff --git a/doc/dA.txt b/doc/dA.txt index fcd5f452..800a807a 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -41,78 +41,81 @@ Autoencoders See section 4.6 of [Bengio09]_ for an overview of auto-encoders. An autoencoder takes an input :math:`\mathbf{x} \in [0,1]^d` and first -maps it (with an *encoder*) to a hidden representation :math:`\mathbf{y} \in [0,1]^{d'}` +maps it (with an *encoder)* to a hidden representation :math:`\mathbf{y} \in [0,1]^{d'}` through a deterministic mapping, e.g.: .. math:: \mathbf{y} = s(\mathbf{W}\mathbf{x} + \mathbf{b}) -Where :math:`s` is a non-linearity such as the sigmoid. -The latent representation :math:`\mathbf{y}`, or **code** is then mapped back (with a *decoder*) into a -**reconstruction** :math:`\mathbf{z}` of same shape as -:math:`\mathbf{x}` through a similar transformation, e.g.: +Where :math:`s` is a non-linearity such as the sigmoid. The latent +representation :math:`\mathbf{y}`, or **code** is then mapped back (with a +*decoder)* into a **reconstruction** :math:`\mathbf{z}` of the same shape as +:math:`\mathbf{x}`. The mapping happens through a similar transformation, e.g.: .. math:: \mathbf{z} = s(\mathbf{W'}\mathbf{y} + \mathbf{b'}) -where ' does not indicate transpose, and -:math:`\mathbf{z}` should be seen as a prediction of :math:`\mathbf{x}`, given the code :math:`\mathbf{y}`. -The weight matrix :math:`\mathbf{W'}` of the reverse mapping may be -optionally constrained by :math:`\mathbf{W'} = \mathbf{W}^T`, which is -an instance of *tied weights*. The parameters of this model (namely -:math:`\mathbf{W}`, :math:`\mathbf{b}`, -:math:`\mathbf{b'}` and, if one doesn't use tied weights, also -:math:`\mathbf{W'}`) are optimized such that the average reconstruction -error is minimized. The reconstruction error can be measured in many ways, depending -on the appropriate distributional assumptions on the input given the code, e.g., using the -traditional *squared error* :math:`L(\mathbf{x}, \mathbf{z}) = || \mathbf{x} - \mathbf{z} ||^2`, -or if the input is interpreted as either bit vectors or vectors of -bit probabilities by the reconstruction *cross-entropy* defined as : +(Here, the prime symbol does *not necessarily* indicate matrix transposition.) +:math:`\mathbf{z}` should be seen as a prediction of :math:`\mathbf{x}`, given +the code :math:`\mathbf{y}`. Optionally, the weight matrix :math:`\mathbf{W'}` +of the reverse mapping may be constrained to be the transpose of the forward +mapping: :math:`\mathbf{W'} = \mathbf{W}^T`. This is referred to as *tied +weights*. The parameters of this model (namely :math:`\mathbf{W}`, +:math:`\mathbf{b}`, :math:`\mathbf{b'}` and, if one doesn't use tied weights, +also :math:`\mathbf{W'}`) are optimized such that the average reconstruction +error is minimized. + +The reconstruction error can be measured in many ways, depending on the +appropriate distributional assumptions on the input given the code. The +traditional *squared error* :math:`L(\mathbf{x} \mathbf{z}) = || \mathbf{x} - +\mathbf{z} ||^2`, can be used. If the input is interpreted as either bit +vectors or vectors of bit probabilities, *cross-entropy* of the reconstruction +can be used: .. math:: L_{H} (\mathbf{x}, \mathbf{z}) = - \sum^d_{k=1}[\mathbf{x}_k \log \mathbf{z}_k + (1 - \mathbf{x}_k)\log(1 - \mathbf{z}_k)] -The hope is that the code :math:`\mathbf{y}` is a distributed representation -that captures the coordinates along the main factors of variation in the data -(similarly to how the projection on principal components captures the main factors -of variation in the data). -Because :math:`\mathbf{y}` is viewed as a lossy compression of :math:`\mathbf{x}`, it cannot -be a good compression (with small loss) for all :math:`\mathbf{x}`, so learning -drives it to be one that is a good compression in particular for training -examples, and hopefully for others as well, but not for arbitrary inputs. -That is the sense in which an auto-encoder generalizes: it gives low reconstruction -error to test examples from the same distribution as the training examples, -but generally high reconstruction error to uniformly chosen configurations of the -input vector. - -If there is one linear hidden layer (the code) and -the mean squared error criterion is used to train the network, then the :math:`k` -hidden units learn to project the input in the span of the first :math:`k` -principal components of the data. If the hidden -layer is non-linear, the auto-encoder behaves differently from PCA, -with the ability to capture multi-modal aspects of the input -distribution. The departure from PCA becomes even more important when -we consider *stacking multiple encoders* (and their corresponding decoders) -when building a deep auto-encoder [Hinton06]_. - -We want to implement an auto-encoder using Theano, in the form of a class, -that could be afterwards used in constructing a stacked autoencoder. The -first step is to create shared variables for the parameters of the -autoencoder ( :math:`\mathbf{W}`, :math:`\mathbf{b}` and -:math:`\mathbf{b'}`, since we are using tied weights in this tutorial ): +The hope is that the code :math:`\mathbf{y}` is a *distributed* representation +that captures the coordinates along the main factors of variation in the data. +This is similar to the way the projection on principal components would capture +the main factors of variation in the data. Indeed, if there is one linear +hidden layer (the *code)* and the mean squared error criterion is used to train +the network, then the :math:`k` hidden units learn to project the input in the +span of the first :math:`k` principal components of the data. If the hidden +layer is non-linear, the auto-encoder behaves differently from PCA, with the +ability to capture multi-modal aspects of the input distribution. The departure +from PCA becomes even more important when we consider *stacking multiple +encoders* (and their corresponding decoders) when building a deep auto-encoder +[Hinton06]_. + +Because :math:`\mathbf{y}` is viewed as a lossy compression of +:math:`\mathbf{x}`, it cannot be a good (small-loss) compression for all +:math:`\mathbf{x}`. Optimization makes it a good compression for training +examples, and hopefully for other inputs as well, but not for arbitrary inputs. +That is the sense in which an auto-encoder generalizes: it gives low +reconstruction error on test examples from the same distribution as the +training examples, but generally high reconstruction error on samples randomly +chosen from the input space. + +We want to implement an auto-encoder using Theano, in the form of a class, that +could be afterwards used in constructing a stacked autoencoder. The first step +is to create shared variables for the parameters of the autoencoder +:math:`\mathbf{W}`, :math:`\mathbf{b}` and :math:`\mathbf{b'}`. (Since we are +using tied weights in this tutorial, :math:`\mathbf{W}^T` will be used for +:math:`\mathbf{W'}`): .. literalinclude:: ../code/dA.py :start-after: start-snippet-1 :end-before: end-snippet-1 -Note that we pass the symbolic ``input`` to the autoencoder as a -parameter. This is such that later we can concatenate layers of -autoencoders to form a deep network: the symbolic output (the :math:`\mathbf{y}` above) of -the k-th layer will be the symbolic input of the (k+1)-th. +Note that we pass the symbolic ``input`` to the autoencoder as a parameter. +This is so that we can concatenate layers of autoencoders to form a deep +network: the symbolic output (the :math:`\mathbf{y}` above) of layer :math:`k` will +be the symbolic input of layer :math:`k+1`. Now we can express the computation of the latent representation and of the reconstructed signal: @@ -137,45 +140,41 @@ reconstruction cost is approximately minimized. :start-after: theano_rng = RandomStreams(rng.randint(2 ** 30)) :end-before: start_time = time.clock() -One serious potential issue with auto-encoders is that if there is no other -constraint besides minimizing the reconstruction error, -then an auto-encoder with :math:`n` inputs and an -encoding of dimension at least :math:`n` could potentially just learn -the identity function, for which many encodings would be useless (e.g., -just copying the input), i.e., the autoencoder would not differentiate -test examples (from the training distribution) from other input configurations. -Surprisingly, experiments reported in [Bengio07]_ nonetheless -suggest that in practice, when trained with -stochastic gradient descent, non-linear auto-encoders with more hidden units -than inputs (called overcomplete) yield useful representations -(in the sense of classification error measured on a network taking this -representation in input). A simple explanation is based on the -observation that stochastic gradient -descent with early stopping is similar to an L2 regularization of the -parameters. To achieve perfect reconstruction of continuous -inputs, a one-hidden layer auto-encoder with non-linear hidden units -(exactly like in the above code) -needs very small weights in the first (encoding) layer (to bring the non-linearity of -the hidden units in their linear regime) and very large weights in the -second (decoding) layer. -With binary inputs, very large weights are -also needed to completely minimize the reconstruction error. Since the -implicit or explicit regularization makes it difficult to reach -large-weight solutions, the optimization algorithm finds encodings which -only work well for examples similar to those in the training set, which is -what we want. It means that the representation is exploiting statistical -regularities present in the training set, rather than learning to -replicate the identity function. - -There are different ways that an auto-encoder with more hidden units -than inputs could be prevented from learning the identity, and still -capture something useful about the input in its hidden representation. -One is the addition of sparsity (forcing many of the hidden units to -be zero or near-zero), and it has been exploited very successfully -by many [Ranzato07]_ [Lee08]_. Another is to add randomness in the transformation from -input to reconstruction. This is exploited in Restricted Boltzmann -Machines (discussed later in :ref:`rbm`), as well as in -Denoising Auto-Encoders, discussed below. +If there is no constraint besides minimizing the reconstruction error, one +might expect an auto-encoder with :math:`n` inputs and an encoding of dimension +:math:`n` (or greater) to learn the identity function, merely mapping an input +to its copy. Such an autoencoder would not differentiate test examples (from +the training distribution) from other input configurations. + +Surprisingly, +experiments reported in [Bengio07]_ suggest that, in practice, when trained +with stochastic gradient descent, non-linear auto-encoders with more hidden +units than inputs (called overcomplete) yield useful representations. (Here, +"useful" means that a network taking the encoding as input has low +classification error.) + +A simple explanation is that stochastic gradient descent with early stopping is +similar to an L2 regularization of the parameters. To achieve perfect +reconstruction of continuous inputs, a one-hidden layer auto-encoder with +non-linear hidden units (exactly like in the above code) needs very small +weights in the first (encoding) layer, to bring the non-linearity of the hidden +units into their linear regime, and very large weights in the second (decoding) +layer. With binary inputs, very large weights are also needed to completely +minimize the reconstruction error. Since the implicit or explicit +regularization makes it difficult to reach large-weight solutions, the +optimization algorithm finds encodings which only work well for examples +similar to those in the training set, which is what we want. It means that the +*representation is exploiting statistical regularities present in the training +set,* rather than merely learning to replicate the input. + +There are other ways by which an auto-encoder with more hidden units than inputs +could be prevented from learning the identity function, capturing something +useful about the input in its hidden representation. One is the addition of +*sparsity* (forcing many of the hidden units to be zero or near-zero). Sparsity +has been exploited very successfully by many [Ranzato07]_ [Lee08]_. Another is +to add randomness in the transformation from input to reconstruction. This +technique is used in Restricted Boltzmann Machines (discussed later in +:ref:`rbm`), as well as in Denoising Auto-Encoders, discussed below. .. _DA: @@ -188,27 +187,23 @@ from simply learning the identity, we train the autoencoder to *reconstruct the input from a corrupted version of it*. The denoising auto-encoder is a stochastic version of the auto-encoder. -Intuitively, a denoising auto-encoder does two things: try to encode the -input (preserve the information about the input), and try to undo the -effect of a corruption process stochastically applied to the input of the -auto-encoder. The latter can only be done by capturing the statistical -dependencies between the inputs. The denoising -auto-encoder can be understood from different perspectives -( the manifold learning perspective, -stochastic operator perspective, -bottom-up -- information theoretic perspective, -top-down -- generative model perspective ), all of which are explained in -[Vincent08]. -See also section 7.2 of [Bengio09]_ for an overview of auto-encoders. - -In [Vincent08], the stochastic corruption process -consists in randomly setting some of the inputs (as many as half of them) -to zero. Hence the denoising auto-encoder is trying to *predict the corrupted (i.e. missing) -values from the uncorrupted (i.e., non-missing) values*, for randomly selected subsets of -missing patterns. Note how being able to predict any subset of variables -from the rest is a sufficient condition for completely capturing the -joint distribution between a set of variables (this is how Gibbs -sampling works). +Intuitively, a denoising auto-encoder does two things: try to encode the input +(preserve the information about the input), and try to undo the effect of a +corruption process stochastically applied to the input of the auto-encoder. The +latter can only be done by capturing the statistical dependencies between the +inputs. The denoising auto-encoder can be understood from different +perspectives ( the manifold learning perspective, stochastic operator +perspective, bottom-up -- information theoretic perspective, top-down -- +generative model perspective ), all of which are explained in [Vincent08]_. See +also section 7.2 of [Bengio09]_ for an overview of auto-encoders. + +In [Vincent08]_, the stochastic corruption process randomly sets some of the +inputs (as many as half of them) to zero. Hence the denoising auto-encoder is +trying to *predict the corrupted (i.e. missing) values from the uncorrupted +(i.e., non-missing) values*, for randomly selected subsets of missing patterns. +Note how being able to predict any subset of variables from the rest is a +sufficient condition for completely capturing the joint distribution between a +set of variables (this is how Gibbs sampling works). To convert the autoencoder class into a denoising autoencoder class, all we need to do is to add a stochastic corruption step operating on the input. The input can be @@ -236,11 +231,11 @@ does just that : -In the stacked autoencoder class (:ref:`stacked_autoencoders`) the -weights of the ``dA`` class have to be shared with those of an -corresponding sigmoid layer. For this reason, the constructor of the ``dA`` also gets Theano -variables pointing to the shared parameters. If those parameters are left -to ``None``, new ones will be constructed. +In the stacked autoencoder class (:ref:`stacked_autoencoders`) the weights of +the ``dA`` class have to be shared with those of a corresponding sigmoid layer. +For this reason, the constructor of the ``dA`` also gets Theano variables +pointing to the shared parameters. If those parameters are left to ``None``, +new ones will be constructed. The final denoising autoencoder class becomes : @@ -465,14 +460,14 @@ it. print ('Training took %f minutes' % (pretraining_time / 60.)) In order to get a feeling of what the network learned we are going to -plot the filters (defined by the weight matrix). Bare in mind however, +plot the filters (defined by the weight matrix). Bear in mind, however, that this does not provide the entire story, since we neglect the biases and plot the weights up to a multiplicative constant (weights are converted to values between 0 and 1). To plot our filters we will need the help of ``tile_raster_images`` (see -:ref:`how-to-plot`) so we urge the reader to familiarize himself with -it. Also using the help of PIL library, the following lines of code will +:ref:`how-to-plot`) so we urge the reader to familiarize himself with it. Also +using the help of the Python Image Library, the following lines of code will save the filters as an image : .. code-block:: python From 2e58c8b76010e9de953705682f4927758f2395a0 Mon Sep 17 00:00:00 2001 From: Jae-Myoung Yu Date: Fri, 17 Oct 2014 17:36:51 +0900 Subject: [PATCH 064/417] print_function and pep8 --- doc/scripts/docgen.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/scripts/docgen.py b/doc/scripts/docgen.py index 8f746e98..a584bcb1 100644 --- a/doc/scripts/docgen.py +++ b/doc/scripts/docgen.py @@ -1,8 +1,7 @@ - +from __future__ import print_function import sys import os import shutil -import inspect import getopt from collections import defaultdict @@ -12,13 +11,14 @@ throot = "/".join(sys.path[0].split("/")[:-2]) options = defaultdict(bool) - options.update(dict([x, y or True] for x, y in getopt.getopt(sys.argv[1:], 'o:', ['rst', 'help', 'nopdf'])[0])) + output_arg = getopt.getopt(sys.argv[1:], 'o:', ['rst', 'help', 'nopdf'])[0] + options.update(dict([x, y or True] for x, y in output_arg)) if options['--help']: - print 'Usage: %s [OPTIONS]' % sys.argv[0] - print ' -o : output the html files in the specified dir' - print ' --rst: only compile the doc (requires sphinx)' - print ' --nopdf: do not produce a PDF file from the doc, only HTML' - print ' --help: this help' + print('Usage: %s [OPTIONS]' % sys.argv[0]) + print(' -o : output the html files in the specified dir') + print(' --rst: only compile the doc (requires sphinx)') + print(' --nopdf: do not produce a PDF file from the doc, only HTML') + print(' --help: this help') sys.exit(0) options['--all'] = not bool(options['--rst']) @@ -49,7 +49,7 @@ def mkdir(path): import tempfile workdir = tempfile.mkdtemp() sphinx.main(['', '-E', '-b', 'latex', - os.path.join(throot, 'doc'), workdir]) + os.path.join(throot, 'doc'), workdir]) # Compile to PDF os.chdir(workdir) os.system('make') @@ -57,10 +57,7 @@ def mkdir(path): shutil.copy(os.path.join(workdir, 'deeplearning.pdf'), outdir) os.chdir(outdir) shutil.rmtree(workdir) - except OSError, e: - print 'OSError:', e - except IOError, e: - print 'IOError:', e - - - + except OSError as e: + print('OSError:', e) + except IOError as e: + print('IOError:', e) From 857473831054969bf1bff6bdbc619a7ff7d9eade Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Tue, 21 Oct 2014 04:03:09 -0700 Subject: [PATCH 065/417] Edits suggested by @lamblin. --- doc/dA.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/dA.txt b/doc/dA.txt index 800a807a..cfdbf080 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -57,7 +57,7 @@ representation :math:`\mathbf{y}`, or **code** is then mapped back (with a \mathbf{z} = s(\mathbf{W'}\mathbf{y} + \mathbf{b'}) -(Here, the prime symbol does *not necessarily* indicate matrix transposition.) +(Here, the prime symbol does not indicate matrix transposition.) :math:`\mathbf{z}` should be seen as a prediction of :math:`\mathbf{x}`, given the code :math:`\mathbf{y}`. Optionally, the weight matrix :math:`\mathbf{W'}` of the reverse mapping may be constrained to be the transpose of the forward @@ -192,9 +192,9 @@ Intuitively, a denoising auto-encoder does two things: try to encode the input corruption process stochastically applied to the input of the auto-encoder. The latter can only be done by capturing the statistical dependencies between the inputs. The denoising auto-encoder can be understood from different -perspectives ( the manifold learning perspective, stochastic operator +perspectives (the manifold learning perspective, stochastic operator perspective, bottom-up -- information theoretic perspective, top-down -- -generative model perspective ), all of which are explained in [Vincent08]_. See +generative model perspective), all of which are explained in [Vincent08]_. See also section 7.2 of [Bengio09]_ for an overview of auto-encoders. In [Vincent08]_, the stochastic corruption process randomly sets some of the From 70e0f7d1f0cd3268fcc9222c960bb1c8a1122a76 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Tue, 21 Oct 2014 13:07:35 -0400 Subject: [PATCH 066/417] Fix size in convnet comment Reported on theano-users --- code/convolutional_mlp.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 895c54e9..ffa2dc7b 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -191,8 +191,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, ) # the HiddenLayer being fully-connected, it operates on 2D matrices of - # shape (batch_size,num_pixels) (i.e matrix of rasterized images). - # This will generate a matrix of shape (20,32*4*4) = (20,512) + # shape (batch_size, num_pixels) (i.e matrix of rasterized images). + # This will generate a matrix of shape (batch_size, nkerns[1] * 4 * 4), + # or (500, 50 * 4 * 4) = (500, 800) with the default values. layer2_input = layer1.output.flatten(2) # construct a fully-connected sigmoidal layer From cee690d9a9913d3916106001fadfc9b1bae67320 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Tue, 21 Oct 2014 13:47:40 -0400 Subject: [PATCH 067/417] Fix a bunch of flake8/pep8 errors --- code/DBN.py | 8 ++-- code/SdA.py | 10 ++--- code/cA.py | 17 ++++---- code/convolutional_mlp.py | 29 ++++++------- code/dA.py | 4 +- code/hmc/hmc.py | 90 ++++++++++++++++++++------------------- code/hmc/test_hmc.py | 4 +- code/logistic_cg.py | 15 +++---- code/logistic_sgd.py | 17 ++++---- code/mlp.py | 3 -- code/rbm.py | 2 - code/rnnrbm.py | 10 ++--- code/test.py | 9 ++-- code/utils.py | 2 +- 14 files changed, 101 insertions(+), 119 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index eacbbee4..8f9715f2 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -1,7 +1,5 @@ """ """ -import cPickle -import gzip import os import sys import time @@ -372,7 +370,6 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, # on the validation set; in this case we # check every epoch - best_params = None best_validation_loss = numpy.inf test_score = 0. start_time = time.clock() @@ -430,9 +427,10 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, end_time = time.clock() print( ( - 'Optimization complete with best validation score of %f %%,' + 'Optimization complete with best validation score of %f %%, ' + 'obtained at iteration %i, ' 'with test performance %f %%' - ) % (best_validation_loss * 100., test_score * 100.) + ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.) ) print >> sys.stderr, ('The fine tuning code for file ' + os.path.split(__file__)[1] + diff --git a/code/SdA.py b/code/SdA.py index 34d5678b..fafa73b5 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -29,8 +29,6 @@ Systems 19, 2007 """ -import cPickle -import gzip import os import sys import time @@ -202,8 +200,6 @@ def pretraining_functions(self, train_set_x, batch_size): index = T.lscalar('index') # index to a minibatch corruption_level = T.scalar('corruption') # % of corruption to use learning_rate = T.scalar('lr') # learning rate to use - # number of batches - n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # begining of a batch, given `index` batch_begin = index * batch_size # ending of a batch given `index` @@ -429,7 +425,6 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, # on the validation set; in this case we # check every epoch - best_params = None best_validation_loss = numpy.inf test_score = 0. start_time = time.clock() @@ -479,10 +474,11 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, end_time = time.clock() print( ( - 'Optimization complete with best validation score of %f %%,' + 'Optimization complete with best validation score of %f %%, ' + 'on iteration %i, ' 'with test performance %f %%' ) - % (best_validation_loss * 100., test_score * 100.) + % (best_validation_loss * 100., best_iter + 1, test_score * 100.) ) print >> sys.stderr, ('The training code for file ' + os.path.split(__file__)[1] + diff --git a/code/cA.py b/code/cA.py index c4e7874a..c7ccd2b0 100644 --- a/code/cA.py +++ b/code/cA.py @@ -12,7 +12,8 @@ squared Frobenius norm of the Jacobian of the hidden mapping h with respect to the visible units yields the contractive auto-encoder: - - \sum_{k=1}^d[ x_k \log z_k + (1-x_k) \log( 1-z_k)] + \| \frac{\partial h(x)}{\partial x} \|^2 + - \sum_{k=1}^d[ x_k \log z_k + (1-x_k) \log( 1-z_k)] + + \| \frac{\partial h(x)}{\partial x} \|^2 References : - S. Rifai, P. Vincent, X. Muller, X. Glorot, Y. Bengio: Contractive @@ -27,8 +28,6 @@ Systems 19, 2007 """ -import cPickle -import gzip import os import sys import time @@ -79,11 +78,11 @@ class cA(object): def __init__(self, numpy_rng, input=None, n_visible=784, n_hidden=100, n_batchsize=1, W=None, bhid=None, bvis=None): - """Initialize the cA class by specifying the number of visible units (the - dimension d of the input ), the number of hidden units ( the dimension - d' of the latent or hidden space ) and the contraction level. The - constructor also receives symbolic variables for the input, weights and - bias. + """Initialize the cA class by specifying the number of visible units + (the dimension d of the input), the number of hidden units (the + dimension d' of the latent or hidden space) and the contraction level. + The constructor also receives symbolic variables for the input, weights + and bias. :type numpy_rng: numpy.random.RandomState :param numpy_rng: number random generator used to generate weights @@ -161,7 +160,7 @@ def __init__(self, numpy_rng, input=None, n_visible=784, n_hidden=100, self.W_prime = self.W.T # if no input is given, generate a variable representing the input - if input == None: + if input is None: # we use a matrix because we expect a minibatch of several # examples, each example being a row self.x = T.dmatrix(name='input') diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index ffa2dc7b..15b98a98 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -21,8 +21,6 @@ http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf """ -import cPickle -import gzip import os import sys import time @@ -53,14 +51,14 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): :type filter_shape: tuple or list of length 4 :param filter_shape: (number of filters, num input feature maps, - filter height,filter width) + filter height, filter width) :type image_shape: tuple or list of length 4 :param image_shape: (batch size, num input feature maps, image height, image width) :type poolsize: tuple or list of length 2 - :param poolsize: the downsampling (pooling) factor (#rows,#cols) + :param poolsize: the downsampling (pooling) factor (#rows, #cols) """ assert image_shape[1] == filter_shape[1] @@ -104,7 +102,7 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): ) # add the bias term. Since the bias is a vector (1D array), we first - # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will + # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) @@ -155,21 +153,21 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels - ishape = (28, 28) # this is the size of MNIST images ###################### # BUILD ACTUAL MODEL # ###################### print '... building the model' - # Reshape matrix of rasterized images of shape (batch_size,28*28) + # Reshape matrix of rasterized images of shape (batch_size, 28 * 28) # to a 4D tensor, compatible with our LeNetConvPoolLayer + # (28, 28) is the size of MNIST images. layer0_input = x.reshape((batch_size, 1, 28, 28)) # Construct the first convolutional pooling layer: - # filtering reduces the image size to (28-5+1,28-5+1)=(24,24) - # maxpooling reduces this further to (24/2,24/2) = (12,12) - # 4D output tensor is thus of shape (batch_size,nkerns[0],12,12) + # filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24) + # maxpooling reduces this further to (24/2, 24/2) = (12, 12) + # 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12) layer0 = LeNetConvPoolLayer( rng, input=layer0_input, @@ -179,9 +177,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, ) # Construct the second convolutional pooling layer - # filtering reduces the image size to (12-5+1,12-5+1)=(8,8) - # maxpooling reduces this further to (8/2,8/2) = (4,4) - # 4D output tensor is thus of shape (nkerns[0],nkerns[1],4,4) + # filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8) + # maxpooling reduces this further to (8/2, 8/2) = (4, 4) + # 4D output tensor is thus of shape (nkerns[0], nkerns[1], 4, 4) layer1 = LeNetConvPoolLayer( rng, input=layer0.output, @@ -240,7 +238,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # SGD Since this model has many parameters, it would be tedious to # manually create an update rule for each model parameter. We thus # create the updates list by automatically looping over all - # (params[i],grads[i]) pairs. + # (params[i], grads[i]) pairs. updates = [ (param_i, param_i - learning_rate * grad_i) for param_i, grad_i in zip(params, grads) @@ -273,7 +271,6 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # on the validation set; in this case we # check every epoch - best_params = None best_validation_loss = numpy.inf best_iter = 0 test_score = 0. @@ -331,7 +328,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, end_time = time.clock() print('Optimization complete.') - print('Best validation score of %f %% obtained at iteration %i,' + print('Best validation score of %f %% obtained at iteration %i, ' 'with test performance %f %%' % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) print >> sys.stderr, ('The code for file ' + diff --git a/code/dA.py b/code/dA.py index 6344221a..e1debf7a 100644 --- a/code/dA.py +++ b/code/dA.py @@ -30,8 +30,6 @@ """ -import cPickle -import gzip import os import sys import time @@ -185,7 +183,7 @@ def __init__( self.W_prime = self.W.T self.theano_rng = theano_rng # if no input is given, generate a variable representing the input - if input == None: + if input is None: # we use a matrix because we expect a minibatch of several # examples, each example being a row self.x = T.dmatrix(name='input') diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index b7c292c0..b9c872f0 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -8,8 +8,8 @@ from theano import tensor as TT import theano -sharedX = lambda X, name: \ - shared(numpy.asarray(X, dtype=theano.config.floatX), name=name) +sharedX = (lambda X, name: + shared(numpy.asarray(X, dtype=theano.config.floatX), name=name)) def kinetic_energy(vel): @@ -145,13 +145,14 @@ def leapfrog(pos, vel, step): # perform leapfrog updates: the scan op is used to repeatedly compute # vel(t + (m-1/2)*stepsize) and pos(t + m*stepsize) for m in [2,n_steps]. - (all_pos, all_vel), scan_updates = theano.scan(leapfrog, - outputs_info=[ - dict(initial=pos_full_step), - dict(initial=vel_half_step), - ], - non_sequences=[stepsize], - n_steps=n_steps - 1) + (all_pos, all_vel), scan_updates = theano.scan( + leapfrog, + outputs_info=[ + dict(initial=pos_full_step), + dict(initial=vel_half_step), + ], + non_sequences=[stepsize], + n_steps=n_steps - 1) final_pos = all_pos[-1] final_vel = all_vel[-1] # NOTE: Scan always returns an updates dictionary, in case the @@ -171,6 +172,7 @@ def leapfrog(pos, vel, step): # return new proposal state return final_pos, final_vel + # start-snippet-1 def hmc_move(s_rng, positions, energy_fn, stepsize, n_steps): """ @@ -224,10 +226,11 @@ def hmc_move(s_rng, positions, energy_fn, stepsize, n_steps): # end-snippet-4 return accept, final_pos + # start-snippet-5 def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, - target_acceptance_rate, stepsize_inc, stepsize_dec, - stepsize_min, stepsize_max, avg_acceptance_slowness): + target_acceptance_rate, stepsize_inc, stepsize_dec, + stepsize_min, stepsize_max, avg_acceptance_slowness): """This function is executed after `n_steps` of HMC sampling (`hmc_move` function). It creates the updates dictionary used by the `simulate` function. It takes care of updating: the position @@ -293,14 +296,15 @@ def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, # perform exponential moving average mean_dtype = theano.scalar.upcast(accept.dtype, avg_acceptance_rate.dtype) new_acceptance_rate = TT.add( - avg_acceptance_slowness * avg_acceptance_rate, - (1.0 - avg_acceptance_slowness) * accept.mean(dtype=mean_dtype)) + avg_acceptance_slowness * avg_acceptance_rate, + (1.0 - avg_acceptance_slowness) * accept.mean(dtype=mean_dtype)) # end-snippet-6 start-snippet-8 return [(positions, new_positions), (stepsize, new_stepsize), (avg_acceptance_rate, new_acceptance_rate)] # end-snippet-8 + class HMC_sampler(object): """ Convenience wrapper for performing Hybrid Monte Carlo (HMC). It creates the @@ -322,11 +326,11 @@ def __init__(self, **kwargs): @classmethod def new_from_shared_positions( - cls, - shared_positions, + cls, + shared_positions, energy_fn, - initial_stepsize=0.01, - target_acceptance_rate=.9, + initial_stepsize=0.01, + target_acceptance_rate=.9, n_steps=20, stepsize_dec=0.98, stepsize_min=0.001, @@ -350,8 +354,6 @@ def new_from_shared_positions( sampling to work. """ - batchsize = shared_positions.shape[0] - # allocate shared variables stepsize = sharedX(initial_stepsize, 'hmc_stepsize') avg_acceptance_rate = sharedX(target_acceptance_rate, @@ -360,40 +362,40 @@ def new_from_shared_positions( # define graph for an `n_steps` HMC simulation accept, final_pos = hmc_move( - s_rng, - shared_positions, - energy_fn, - stepsize, - n_steps) + s_rng, + shared_positions, + energy_fn, + stepsize, + n_steps) # define the dictionary of updates, to apply on every `simulate` call simulate_updates = hmc_updates( - shared_positions, - stepsize, - avg_acceptance_rate, - final_pos=final_pos, - accept=accept, - stepsize_min=stepsize_min, - stepsize_max=stepsize_max, - stepsize_inc=stepsize_inc, - stepsize_dec=stepsize_dec, - target_acceptance_rate=target_acceptance_rate, - avg_acceptance_slowness=avg_acceptance_slowness) + shared_positions, + stepsize, + avg_acceptance_rate, + final_pos=final_pos, + accept=accept, + stepsize_min=stepsize_min, + stepsize_max=stepsize_max, + stepsize_inc=stepsize_inc, + stepsize_dec=stepsize_dec, + target_acceptance_rate=target_acceptance_rate, + avg_acceptance_slowness=avg_acceptance_slowness) # compile theano function simulate = function([], [], updates=simulate_updates) # create HMC_sampler object with the following attributes ... return cls( - positions=shared_positions, - stepsize=stepsize, - stepsize_min=stepsize_min, - stepsize_max=stepsize_max, - avg_acceptance_rate=avg_acceptance_rate, - target_acceptance_rate=target_acceptance_rate, - s_rng=s_rng, - _updates=simulate_updates, - simulate=simulate) + positions=shared_positions, + stepsize=stepsize, + stepsize_min=stepsize_min, + stepsize_max=stepsize_max, + avg_acceptance_rate=avg_acceptance_rate, + target_acceptance_rate=target_acceptance_rate, + s_rng=s_rng, + _updates=simulate_updates, + simulate=simulate) def draw(self, **kwargs): """ diff --git a/code/hmc/test_hmc.py b/code/hmc/test_hmc.py index 2f672b22..0a70190a 100644 --- a/code/hmc/test_hmc.py +++ b/code/hmc/test_hmc.py @@ -28,7 +28,7 @@ def gaussian_energy(x): # Create HMC sampler sampler = sampler_cls(position, gaussian_energy, - initial_stepsize=1e-3, stepsize_max=0.5) + initial_stepsize=1e-3, stepsize_max=0.5) # Start with a burn-in process garbage = [sampler.draw() for r in xrange(burnin)] # burn-in Draw @@ -55,7 +55,7 @@ def gaussian_energy(x): def test_hmc(): sampler = sampler_on_nd_gaussian(HMC_sampler.new_from_shared_positions, - burnin=1000, n_samples=1000, dim=5) + burnin=1000, n_samples=1000, dim=5) assert abs(sampler.avg_acceptance_rate.get_value() - sampler.target_acceptance_rate) < .1 assert sampler.stepsize.get_value() >= sampler.stepsize_min diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 2540c038..05f562a1 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -22,9 +22,8 @@ y_{pred} = argmax_i P(Y=i|x,W,b) -This tutorial presents a stochastic gradient descent optimization method -suitable for large datasets, and a conjugate gradient optimization method -that is suitable for smaller datasets. +This tutorial presents a conjugate gradient optimization method that is +suitable for smaller datasets. References: @@ -37,8 +36,6 @@ __docformat__ = 'restructedtext en' -import cPickle -import gzip import os import sys import time @@ -107,8 +104,9 @@ def negative_log_likelihood(self, y): .. math:: \frac{1}{|\mathcal{D}|}\mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = - \frac{1}{|\mathcal{D}|}\sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ - \ell (\theta=\{W,b\}, \mathcal{D}) + \frac{1}{|\mathcal{D}|}\sum_{i=0}^{|\mathcal{D}|} + \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ + \ell (\theta=\{W,b\}, \mathcal{D}) :type y: theano.tensor.TensorType :param y: corresponds to a vector that gives for each example the @@ -129,7 +127,7 @@ def errors(self, y): if y.ndim != self.y_pred.ndim: raise TypeError( 'y should have the same shape as self.y_pred', - ('y', target.type, 'y_pred', self.y_pred.type) + ('y', y.type, 'y_pred', self.y_pred.type) ) # check if y is of the correct datatype if y.dtype.startswith('int'): @@ -168,7 +166,6 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size - ishape = (28, 28) # this is the size of MNIST images n_in = 28 * 28 # number of input units n_out = 10 # number of output units diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 6904717e..599f5658 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -23,8 +23,7 @@ This tutorial presents a stochastic gradient descent optimization method -suitable for large datasets, and a conjugate gradient optimization method -that is suitable for smaller datasets. +suitable for large datasets. References: @@ -106,7 +105,7 @@ def __init__(self, input, n_in, n_out): # probability is maximal self.y_pred = T.argmax(self.p_y_given_x, axis=1) # end-snippet-1 - + # parameters of the model self.params = [self.W, self.b] @@ -117,8 +116,9 @@ def negative_log_likelihood(self, y): .. math:: \frac{1}{|\mathcal{D}|} \mathcal{L} (\theta=\{W,b\}, \mathcal{D}) = - \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ - \ell (\theta=\{W,b\}, \mathcal{D}) + \frac{1}{|\mathcal{D}|} \sum_{i=0}^{|\mathcal{D}|} + \log(P(Y=y^{(i)}|x^{(i)}, W,b)) \\ + \ell (\theta=\{W,b\}, \mathcal{D}) :type y: theano.tensor.TensorType :param y: corresponds to a vector that gives for each example the @@ -155,7 +155,7 @@ def errors(self, y): if y.ndim != self.y_pred.ndim: raise TypeError( 'y should have the same shape as self.y_pred', - ('y', target.type, 'y_pred', self.y_pred.type) + ('y', y.type, 'y_pred', self.y_pred.type) ) # check if y is of the correct datatype if y.dtype.startswith('int'): @@ -358,7 +358,6 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # on the validation set; in this case we # check every epoch - best_params = None best_validation_loss = numpy.inf test_score = 0. start_time = time.clock() @@ -405,8 +404,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, print( ( - ' epoch %i, minibatch %i/%i, test error of best' - ' model %f %%' + ' epoch %i, minibatch %i/%i, test error of' + ' best model %f %%' ) % ( epoch, diff --git a/code/mlp.py b/code/mlp.py index 6f2fb92e..6701378c 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -21,8 +21,6 @@ __docformat__ = 'restructedtext en' -import cPickle -import gzip import os import sys import time @@ -335,7 +333,6 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # on the validation set; in this case we # check every epoch - best_params = None best_validation_loss = numpy.inf best_iter = 0 test_score = 0. diff --git a/code/rbm.py b/code/rbm.py index 875b8dd2..2c821fc9 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -4,8 +4,6 @@ contain hidden variables. Restricted Boltzmann Machines further restrict BMs to those without visible-visible and hidden-hidden connections. """ -import cPickle -import gzip import time try: diff --git a/code/rnnrbm.py b/code/rnnrbm.py index 3b8ede88..e1f40b5a 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -207,11 +207,11 @@ def __init__( self.r = r self.dt = dt (v, v_sample, cost, monitor, params, updates_train, v_t, - updates_generate) = build_rnnrbm( - r[1] - r[0], - n_hidden, - n_hidden_recurrent - ) + updates_generate) = build_rnnrbm( + r[1] - r[0], + n_hidden, + n_hidden_recurrent + ) gradient = T.grad(cost, params, consider_constant=[v_sample]) updates_train.update( diff --git a/code/test.py b/code/test.py index 5cb13c89..30025c00 100644 --- a/code/test.py +++ b/code/test.py @@ -1,7 +1,6 @@ import sys import numpy -import theano import convolutional_mlp import dA @@ -101,7 +100,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 21.5, - 11.8, 47.9, 290.1, 315.4]) + 11.8, 47.9, 290.1, 315.4]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] @@ -169,7 +168,8 @@ def do_tests(): print >> sys.stderr, 'float64/float32', ( float64_times / float32_times) print >> sys.stderr - print >> sys.stderr, 'Duplicate the timing to have everything in one place' + print >> sys.stderr, ('Duplicate the timing to have everything ' + 'in one place') print >> sys.stderr, algo_executed print >> sys.stderr, 'float64 times', float64_times print >> sys.stderr, 'float64 expected', expected_times_64 @@ -201,7 +201,8 @@ def do_tests(): if (do_float64 + do_float32 + do_gpu) > 1: print >> sys.stderr - print >> sys.stderr, 'Duplicate the timing to have everything in one place' + print >> sys.stderr, ('Duplicate the timing to have everything ' + 'in one place') print >> sys.stderr, algo_executed if do_float64: print >> sys.stderr, 'float64 times', float64_times diff --git a/code/utils.py b/code/utils.py index 246c2569..3b50019c 100644 --- a/code/utils.py +++ b/code/utils.py @@ -135,5 +135,5 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), out_array[ tile_row * (H + Hs): tile_row * (H + Hs) + H, tile_col * (W + Ws): tile_col * (W + Ws) + W - ] = this_img * c + ] = this_img * c return out_array From 0b6d1ee4776c42b2bf463435b2289c506613781e Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Mon, 3 Nov 2014 08:55:25 -0800 Subject: [PATCH 068/417] Edits to SdA tutorial for clarity. --- doc/SdA.txt | 107 +++++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/doc/SdA.txt b/doc/SdA.txt index 8caa5c3f..4548bd4f 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -4,7 +4,7 @@ Stacked Denoising Autoencoders (SdA) ==================================== .. note:: - This section assumes the reader has already read through :doc:`logreg` + This section assumes you have already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. @@ -32,8 +32,8 @@ Stacked Denoising Autoencoders (SdA) The Stacked Denoising Autoencoder (SdA) is an extension of the stacked autoencoder [Bengio07]_ and it was introduced in [Vincent08]_. -This tutorial builds on the previous tutorial :ref:`dA` and we recommend, -especially if you do not have experience with autoencoders, to read it +This tutorial builds on the previous tutorial :ref:`dA`. +Especially if you do not have experience with autoencoders, we recommend reading it before going any further. .. _stacked_autoencoders: @@ -41,37 +41,40 @@ before going any further. Stacked Autoencoders ++++++++++++++++++++ -The denoising autoencoders can be stacked to form a deep network by +Denoising autoencoders can be stacked to form a deep network by feeding the latent representation (output code) -of the denoising auto-encoder found on the layer +of the denoising autoencoder found on the layer below as input to the current layer. The **unsupervised pre-training** of such an architecture is done one layer at a time. Each layer is trained as -a denoising auto-encoder by minimizing the reconstruction of its input +a denoising autoencoder by minimizing the error in reconstructing its input (which is the output code of the previous layer). Once the first :math:`k` layers are trained, we can train the :math:`k+1`-th layer because we can now compute the code or latent representation from the layer below. + Once all layers are pre-trained, the network goes through a second stage -of training called **fine-tuning**. Here we consider **supervised fine-tuning** +of training called **fine-tuning**, where we want to minimize prediction error on a supervised task. -For this we first add a logistic regression +For this, we first add a logistic regression layer on top of the network (more precisely on the output code of the output layer). We then train the entire network as we would train a multilayer perceptron. At this point, we only consider the encoding parts of each auto-encoder. This stage is supervised, since now we use the target class during -training (see the :ref:`mlp` for details on the multilayer perceptron). +training. (See the :ref:`mlp` for details on the multilayer perceptron.) This can be easily implemented in Theano, using the class defined -before for a denoising autoencoder. We can see the stacked denoising -autoencoder as having two facades, one is a list of -autoencoders, the other is an MLP. During pre-training we use the first facade, i.e we treat our model +previously for a denoising autoencoder. We can see the stacked denoising +autoencoder as having two facades: One is a list of +autoencoders. The other is an MLP. During pre-training we use the first facade, i.e., we treat our model as a list of autoencoders, and train each autoencoder seperately. In the second stage of training, we use the second facade. These two -facedes are linked by the fact that the autoencoders and the sigmoid layers of -the MLP share parameters, and the fact that autoencoders get as input latent -representations of intermediate layers of the MLP. +facades are linked + +* by the parameters shared by the autoencoders and the sigmoid layers of the MLP, and + +* by feeding the latent representations of intermediate layers of the MLP as input to the autoencoders. .. literalinclude:: ../code/SdA.py :start-after: start-snippet-1 @@ -80,20 +83,20 @@ representations of intermediate layers of the MLP. ``self.sigmoid_layers`` will store the sigmoid layers of the MLP facade, while ``self.dA_layers`` will store the denoising autoencoder associated with the layers of the MLP. -Next step, we construct ``n_layers`` sigmoid layers (we use the -``HiddenLayer`` class introduced in :ref:`mlp`, with the only -modification that we replaced the non-linearity from ``tanh`` to the -logistic function :math:`s(x) = \frac{1}{1+e^{-x}}`) and ``n_layers`` -denoising autoencoders, where ``n_layers`` is the depth of our model. -We link the sigmoid layers such that they form an MLP, and construct -each denoising autoencoder such that they share the weight matrix and the -bias of the encoding part with its corresponding sigmoid layer. +Next, we construct ``n_layers`` denoising autoencoders and ``n_layers`` sigmoid +layers, where ``n_layers`` is the depth of our model. We use the +``HiddenLayer`` class introduced in :ref:`mlp`, with one +modification: we replace the ``tanh`` non-linearity with the +logistic function :math:`s(x) = \frac{1}{1+e^{-x}}`). +We link the sigmoid layers to form an MLP, and construct +the denoising autoencoders such that each shares the weight matrix and the +bias of its encoding part with its corresponding sigmoid layer. .. literalinclude:: ../code/SdA.py :start-after: start-snippet-2 :end-before: end-snippet-2 -All we need now is to add the logistic layer on top of the sigmoid +All we need now is to add a logistic layer on top of the sigmoid layers such that we have an MLP. We will use the ``LogisticRegression`` class introduced in :ref:`logreg`. @@ -101,57 +104,57 @@ use the ``LogisticRegression`` class introduced in :ref:`logreg`. :start-after: end-snippet-2 :end-before: def pretraining_functions -The class also provides a method that generates training functions for -each of the denoising autoencoder associated with the different layers. +The ``SdA`` class also provides a method that generates training functions for +the denoising autoencoders in its layers. They are returned as a list, where element :math:`i` is a function that -implements one step of training the ``dA`` correspoinding to layer +implements one step of training the ``dA`` corresponding to layer :math:`i`. .. literalinclude:: ../code/SdA.py :start-after: self.errors = self.logLayer.errors(self.y) :end-before: corruption_level = T.scalar('corruption') -In order to be able to change the corruption level or the learning rate -during training we associate a Theano variable to them. +To be able to change the corruption level or the learning rate +during training, we associate Theano variables with them. .. literalinclude:: ../code/SdA.py :start-after: index = T.lscalar('index') :end-before: def build_finetune_functions Now any function ``pretrain_fns[i]`` takes as arguments ``index`` and -optionally ``corruption`` -- the corruption level or ``lr`` -- the -learning rate. Note that the name of the parameters are the name given -to the Theano variables when they are constructed, not the name of the -python variables (``learning_rate`` or ``corruption_level``). Keep this +optionally ``corruption``---the corruption level or ``lr``---the +learning rate. Note that the names of the parameters are the names given +to the Theano variables when they are constructed, not the names of the +Python variables (``learning_rate`` or ``corruption_level``). Keep this in mind when working with Theano. -In the same fashion we build a method for constructing function required -during finetuning ( a ``train_model``, a ``validate_model`` and a -``test_model`` function). +In the same fashion we build a method for constructing the functions required +during finetuning (``train_fn``, ``valid_score`` and +``test_score``). .. literalinclude:: ../code/SdA.py :pyobject: SdA.build_finetune_functions -Note that the returned ``valid_score`` and ``test_score`` are not Theano -functions, but rather python functions that also loop over the entire -validation set and the entire test set producing a list of the losses +Note that ``valid_score`` and ``test_score`` are not Theano +functions, but rather Python functions that loop over the entire +validation set and the entire test set, respectively, producing a list of the losses over these sets. Putting it all together +++++++++++++++++++++++ -The few lines of code below constructs the stacked denoising -autoencoder : +The few lines of code below construct the stacked denoising +autoencoder: .. literalinclude:: ../code/SdA.py :start-after: start-snippet-3 :end-before: end-snippet-3 -There are two stages in training this network, a layer-wise pre-training and -fine-tuning afterwards. +There are two stages of training for this network: layer-wise pre-training +followed by fine-tuning. For the pre-training stage, we will loop over all the layers of the -network. For each layer we will use the compiled theano function that +network. For each layer we will use the compiled Theano function that implements a SGD step towards optimizing the weights for reducing the reconstruction cost of that layer. This function will be applied to the training set for a fixed number of epochs given by @@ -161,9 +164,9 @@ to the training set for a fixed number of epochs given by :start-after: start-snippet-4 :end-before: end-snippet-4 -The fine-tuning loop is very similar with the one in the :ref:`mlp`, the -only difference is that we will use now the functions given by -``build_finetune_functions`` . +The fine-tuning loop is very similar to the one in the :ref:`mlp`. The +only difference is that it uses the functions given by +``build_finetune_functions``. Running the Code ++++++++++++++++ @@ -175,8 +178,8 @@ The user can run the code by calling: python code/SdA.py By default the code runs 15 pre-training epochs for each layer, with a batch -size of 1. The corruption level for the first layer is 0.1, for the second -0.2 and 0.3 for the third. The pretraining learning rate is was 0.001 and +size of 1. The corruption levels are 0.1 for the first layer, 0.2 for the second, +and 0.3 for the third. The pretraining learning rate is 0.001 and the finetuning learning rate is 0.1. Pre-training takes 585.01 minutes, with an average of 13 minutes per epoch. Fine-tuning is completed after 36 epochs in 444.2 minutes, with an average of 12.34 minutes per epoch. The final @@ -188,13 +191,13 @@ Xeon E5430 @ 2.66GHz CPU, with a single-threaded GotoBLAS. Tips and Tricks +++++++++++++++ -One way to improve the running time of your code (given that you have +One way to improve the running time of your code (assuming you have sufficient memory available), is to compute how the network, up to layer :math:`k-1`, transforms your data. Namely, you start by training your first layer dA. Once it is trained, you can compute the hidden units values for every datapoint in your dataset and store this as a new dataset that you will -use to train the dA corresponding to layer 2. Once you trained the dA for +use to train the dA corresponding to layer 2. Once you have trained the dA for layer 2, you compute, in a similar fashion, the dataset for layer 3 and so on. You can see now, that at this point, the dAs are trained individually, and they just provide (one to the other) a non-linear transformation of the input. -Once all dAs are trained, you can start fine-tunning the model. +Once all dAs are trained, you can start fine-tuning the model. From af449a93855bc1e65db9325b049258d9386ca9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 11 Feb 2014 17:06:34 -0500 Subject: [PATCH 069/417] modify the index --- doc/contents.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/contents.txt b/doc/contents.txt index 381043d9..c21e7806 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -19,6 +19,7 @@ Contents rbm DBN hmc + rnnslu rnnrbm utilities references From aa258d0b85919edae1b061935b9734944ee2ae6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 11 Feb 2014 17:06:58 -0500 Subject: [PATCH 070/417] rnn with word embeddings tutorial --- doc/rnnslu.txt | 411 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 doc/rnnslu.txt diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt new file mode 100644 index 00000000..eadb1b23 --- /dev/null +++ b/doc/rnnslu.txt @@ -0,0 +1,411 @@ +Recurrent Neural Networks with Word Embeddings +********************************************** + +Summary ++++++++ + +In this tutorial, you will learn how to: + +* learn **Word Embeddings** +* using **Recurrent Neural Networks** architectures +* with **Context Windows** + +in order to perform Semantic Parsing / Slot-Filling (Spoken Language Understanding) + +Code - Citations - Contact +++++++++++++++++++++++++++ + +**Code** + +Directly running experiments is also possible using this `github repository `_. + +**Papers** + +If you use this tutorial, cite the following papers: + +* `[pdf] `_ Grégoire Mesnil, Xiaodong He, Li Deng and Yoshua Bengio. Investigation of Recurrent-Neural-Network Architectures and Learning Methods for Spoken Language Understanding. Interspeech, 2013. + +* `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. + +* Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. + +Thank you! + +**Contact** + +Please email to `Grégoire Mesnil `_ for any +problem report or feedback. We will be glad to hear from you. + +Task +++++ + +The Slot-Filling (Spoken Language Understanding) consists in assigning a label +to each word given a sentence. It's a classification task. + +Dataset ++++++++ + +An old and small benchmark for this task is the ATIS (Airline Travel Information +System) dataset collected by DARPA. Here is a sentence (or utterance) example using the +`Inside Outside Beginning (IOB) +`_ representation. + ++--------------------+------+--------+-----+--------+---+-------+-------+--------+ +| **Input** (words) | show | flights| from| Boston | to| New | York | today | ++--------------------+------+--------+-----+--------+---+-------+-------+--------+ +| **Output** (labels)| O | O | O | B-dept | O | B-arr | I-arr | B-date | ++--------------------+------+--------+-----+--------+---+-------+-------+--------+ + +The ATIS offical split contains 4,978/893 sentences for a total of 56,590/9,198 +words (average sentence length is 15) in the train/test set. The number of +classes (different slots) is 128 including the O label (NULL). + +As `Microsoft Research people +`_, +we deal with unseen words in the test set by marking any words with only one +single occurrence in the training set as ```` and use this token to +represent those unseen words in the test set. As `Ronan Collobert and colleagues +`_, we converted +sequences of numbers with the string ``DIGIT`` i.e. ``1984`` is converted to +``DIGITDIGITDIGITDIGIT``. + +We split the official train set into a training and validation set that contain +respectively 80% and 20% of the official training sentences. `Significant +performance improvement difference has to be greater than 0.6% in F1 measure at +the 95% level due to the small size of the dataset +`_. +For evaluation purpose, experiments have to report the following metrics: + +* `Precision `_ +* `Recall `_ +* `F1 score `_ + +We will use the `conlleval +`_ PERL script to +measure the performance of our models. + +Recurrent Neural Network Model +++++++++++++++++++++++++++++++ + +**Raw input encoding** + +Each token in the ATIS vocabulary is associated to an index. Each sentence is a +array of indexes (``int32``). Each set is a list of arrays of indexes. A python +dictionnary is defined for mapping the space of indexes to the space of words. + + >>> sentence + array([383, 189, 13, 193, 208, 307, 195, 502, 260, 539, + 7, 60, 72, 8, 350, 384], dtype=int32) + >>> map(lambda x: index2word[x], sentence) + ['please', 'find', 'a', 'flight', 'from', 'miami', 'florida', + 'to', 'las', 'vegas', '', 'arriving', 'before', 'DIGIT', "o'clock", 'pm'] + +Same thing for labels corresponding to this particular sentence. + + >>> labels + array([126, 126, 126, 126, 126, 48, 50, 126, 78, 123, 81, 126, 15, + 14, 89, 89], dtype=int32) + >>> map(lambda x: index2label[x], labels) + ['O', 'O', 'O', 'O', 'O', 'B-fromloc.city_name', 'B-fromloc.state_name', + 'O', 'B-toloc.city_name', 'I-toloc.city_name', 'B-toloc.state_name', + 'O', 'B-arrive_time.time_relative', 'B-arrive_time.time', + 'I-arrive_time.time', 'I-arrive_time.time'] + +**Context window** + +Given a sentence i.e. an array of indexes, and a window size i.e. 1,3,5,..., we +need to convert each word in the sentence to a context window surrounding this +particular word. In details, we have:: + + def contextwin(l, win): + ''' + win :: int corresponding to the size of the window + given a list of indexes composing a sentence + + l :: array containing the word indexes + + it will return a list of list of indexes corresponding + to context windows surrounding each word in the sentence + ''' + + assert (win % 2) == 1 + assert win >=1 + l = list(l) + + lpadded = win/2 * [-1] + l + win/2 * [-1] + out = [ lpadded[i:i+win] for i in range(len(l)) ] + + assert len(out) == len(l) + return out + +The index ``-1`` corresponds to the ``PADDING`` index we insert at the +beginning/end of the sentence. + +Here is a sample: + + >>> x + array([0, 1, 2, 3, 4], dtype=int32) + >>> contextwin(x, 3) + [[-1, 0, 1], + [ 0, 1, 2], + [ 1, 2, 3], + [ 2, 3, 4], + [ 3, 4,-1]] + >>> contextwin(x, 7) + [[-1, -1, -1, 0, 1, 2, 3], + [-1, -1, 0, 1, 2, 3, 4], + [-1, 0, 1, 2, 3, 4,-1], + [ 0, 1, 2, 3, 4,-1,-1], + [ 1, 2, 3, 4,-1,-1,-1]] + +To summarize, we started with an array of indexes and ended with a matrix of +indexes. Each line corresponds to the context window surrounding this word. + +**Word embeddings** + +Once we have the sentence converted to context windows i.e. a matrix of indexes, we have to associate +these indexes to the embeddings (real-valued vector associated to each word). +Using Theano, it gives:: + + import theano, numpy + from theano import tensor as T + + # nv :: size of our vocabulary + # de :: dimension of the embedding space + nv, de = 1000, 50 + + embeddings = theano.shared(0.2 * numpy.random.uniform(-1.0, 1.0, \ + (nv+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end + + idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence + x, _ = theano.scan(fn = lambda idx: embeddings[idx].flatten(), sequences = idxs) + +The x symbolic variable corresponds to a matrix of shape (number of words in the +sentences, dimension of the embedding space X context window size). + +Let's compile a theano function to do so + + >>> x + array([0, 1, 2, 3, 4], dtype=int32) + >>> cx = contextwin(x, 7) + [[-1, -1, -1, 0, 1, 2, 3], + [-1, -1, 0, 1, 2, 3, 4], + [-1, 0, 1, 2, 3, 4,-1], + [ 0, 1, 2, 3, 4,-1,-1], + [ 1, 2, 3, 4,-1,-1,-1]] + >>> f = theano.function( inputs=[idxs], outputs=x) + >>> f(cx) + array([[-0.08088442, 0.08458307, 0.05064092, ..., 0.06876887, + -0.06648078, -0.15192257], + [-0.08088442, 0.08458307, 0.05064092, ..., 0.11192625, + 0.08745284, 0.04381778], + [-0.08088442, 0.08458307, 0.05064092, ..., -0.00937143, + 0.10804889, 0.1247109 ], + [ 0.11038255, -0.10563177, -0.18760249, ..., -0.00937143, + 0.10804889, 0.1247109 ], + [ 0.18738101, 0.14727569, -0.069544 , ..., -0.00937143, + 0.10804889, 0.1247109 ]], dtype=float32) + >>> f(cx).shape + (5, 350) + + +We now have a sequence (of length 5 which is corresponds to the length of the +sentence) of **context window word embeddings** which is easy to feed to a simple +recurrent neural network to iterate with. + +**Elman recurrent neural network** + +The followin (Elman) recurrent neural network (E-RNN) takes as input the current input +(time ``t``) and the previous hiddent state (time ``t-1``). Then it iterates. + +In the previous section, we processed the input to fit this +sequential/temporal. It consists in a matrix where the row ``0`` corresponds to +the time step ``t=0``, the row ``1`` corresponds to the time step ``t=1``, etc. + +The **parameters** of the E-RNN to be learned are: + +* the word embeddings (real-valued matrix) +* the initial hidden state (real-value vector) +* two matrices for the linear projection of the input ``t`` and the previous hidden layer state ``t-1`` +* (optionnal) bias. `Recommendation `_: don't use it. +* softmax classification layer on top + +The **hyperparameters** define the whole architecture: + +* dimension of the word embedding +* size of the vocabulary +* number of hidden units +* number of classes +* random seed + way to initialize the model + +It gives the following code:: + + class ERNN(object): + + def __init__(self, nh, nc, ne, de, cs): + ''' + nh :: dimension of the hidden layer + nc :: number of classes + ne :: number of word embeddings in the vocabulary + de :: dimension of the word embeddings + cs :: word window context size + ''' + self.emb = theano.shared(name='embeddings', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + (ne+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end + self.Wx = theano.shared(name='Wx', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + (de * cs, nh)).astype(theano.config.floatX)) + self.Wh = theano.shared(name='Wh', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + (nh, nh)).astype(theano.config.floatX)) + self.W = theano.shared(name='W', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + (nh, nc)).astype(theano.config.floatX)) + self.bh = theano.shared(name='bh', value=numpy.zeros(nh, dtype=theano.config.floatX)) + self.b = theano.shared(name='b', value=numpy.zeros(nc, dtype=theano.config.floatX)) + self.h0 = theano.shared(name='h0', value=numpy.zeros(nh, dtype=theano.config.floatX)) + + # bundle + self.params = [ self.emb, self.Wx, self.Wh, self.W, self.bh, self.b, self.h0 ] + +Then we integrate the way to build the input from the embedding matrix:: + + idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence + x, _ = theano.scan(fn = lambda idx: self.emb[idx].flatten(), sequences = idxs) + y = T.ivector('y') # label + +We use the scan operator to construct the recursion, works like a charm:: + + def recurrence(x_t, h_tm1): + h_t = T.nnet.sigmoid(T.dot(x_t, self.Wx) + T.dot(h_tm1, self.Wh) + self.bh) + s_t = T.nnet.softmax(T.dot(h_t, self.W) + self.b) + return [h_t, s_t] + + [h, s], _ = theano.scan(fn=recurrence, \ + sequences=x, outputs_info=[self.h0, None], \ + n_steps=x.shape[0]) + + p_y_given_x_sentence = s[:,0,:] + y_pred = T.argmax(p_y_given_x_sentence, axis=1) + +Theano will then compute all the gradients automatically to maximize the log-likelihood:: + + lr = T.scalar('lr') + nll = -T.mean(T.log(p_y_given_x_sentence)[T.arange(x.shape[0]),y]) + gradients = T.grad( nll, self.params ) + updates = OrderedDict(( p, p-lr*g ) for p, g in zip( self.params , gradients)) + +Next compile those functions:: + + self.classify = theano.function(inputs=[idxs], outputs=y_pred) + + self.train = theano.function( inputs = [idxs, y, lr], + outputs = nll, + updates = updates ) + +We keep the word embeddings on the unit sphere by normalizing them after each update:: + + self.normalize = theano.function( inputs = [], + updates = {self.emb:\ + self.emb/T.sqrt((self.emb**2).sum(axis=1)).dimshuffle(0,'x')}) + +And that's it! + +Evaluation +++++++++++ + +With the previous defined functions, you can compare the predicted labels with +the true labels and compute some metrics. In this `repo +`_, we build a wrapper around the `conlleval +`_ PERL script. +It's not trivial to compute those metrics due to the `Inside Outside Beginning +(IOB) `_ representation +i.e. a prediction is considered correct if the word-beginnin **and** the +word-inside **and** the word-outside predictions are **all** correct. + +Training +++++++++ + +**Updates** + +For stochastic gradient descent (SGD) update, we consider the whole sentence as a mini-batch +and perform one update per sentence. It is possible to perform a pure SGD (contrary to mini-batch) +where the update is done on only one single word at a time. + +After each iteration/update, we normalize the word embeddings to keep them on a unit sphere. + +**Stopping Criterion** + +Early-stopping on a validation set is our regularization technique: +the training is run for a given number of epochs (a single pass through the +whole dataset) and keep the best model along with respect to the F1 score +computed on the validation set after each epoch. + +**Hyper-Parameter Selection** + +Although there is interesting research/`code +`_ on the topic of automatic +hyper-parameter selection, we use the `KISS +`_ random search. + +The following intervals can give you some starting point: + +* learning rate : uniform([0.05,0.01]) +* window size : random value from {3,...,19} +* number of hidden units : random value from {100,200} +* embedding dimension : random value from {50,100} + +Results ++++++++ + +**Timing** + +Running experiments on ATIS using this `repository `_ +will run one epoch in less than 40 seconds on bart1 processor using less than 200 Mo of RAM:: + + [learning] epoch 0 >> 100.00% completed in 34.48 (sec) << + +After a few epochs, you obtain decent performance **94.48 % of F1 score**.:: + + NEW BEST: epoch 28 valid F1 96.61 best test F1 94.19 + NEW BEST: epoch 29 valid F1 96.63 best test F1 94.42 + [learning] epoch 30 >> 100.00% completed in 35.04 (sec) << + [learning] epoch 31 >> 100.00% completed in 34.80 (sec) << + [...] + NEW BEST: epoch 40 valid F1 97.25 best test F1 94.34 + [learning] epoch 41 >> 100.00% completed in 35.18 (sec) << + NEW BEST: epoch 42 valid F1 97.33 best test F1 94.48 + [learning] epoch 43 >> 100.00% completed in 35.39 (sec) << + [learning] epoch 44 >> 100.00% completed in 35.31 (sec) << + [...] + +**Word Embedding Nearest Neighbors** + +We can check the k-nearest neighbors of the thus learned embeddings. L2 and +cosine distance gave the same results so we plot them for the cosine distance. + ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|**atlanta** |**back** |**ap80** |**but** |**aircraft** |**business** |**a** |**august** |**actually** |**cheap** | ++==============================+==============================+==============================+==============================+==============================+==============================+==============================+==============================+==============================+==============================+ +|phoenix |live |ap57 |if |plane |coach |people |september |provide |weekday | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|denver |lives |ap |up |service |first |do |january |prices |weekdays | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|tacoma |both |connections |a |airplane |fourth |but |june |stop |am | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|columbus |how |tomorrow |now |seating |thrift |numbers |december |number |early | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|seattle |me |before |amount |stand |tenth |abbreviation |november |flight |sfo | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|minneapolis |out |earliest |more |that |second |if |april |there |milwaukee | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|pittsburgh |other |connect |abbreviation |on |fifth |up |july |serving |jfk | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|ontario |plane |thrift |restrictions |turboprop |third |serve |jfk |thank |shortest | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|montreal |service |coach |mean |mean |twelfth |database |october |ticket |bwi | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ +|philadelphia |fare |today |interested |amount |sixth |passengers |may |are |lastest | ++------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ + +As you can judge, the limited size of the vocabulary (about 500 words) gives us mitigated +performance. According to human judgement: some are good, some are bad. + + From 500eb77e432352ea7082253f7dacec976970a7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 9 Apr 2014 12:38:27 -0400 Subject: [PATCH 071/417] add pdf link --- doc/rnnslu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index eadb1b23..98dae465 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -27,7 +27,7 @@ If you use this tutorial, cite the following papers: * `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. -* Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. +* `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. Thank you! From 827c2915f3b9a1a9a0aac5cd37c8e4a622c4a8fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 9 Apr 2014 12:46:47 -0400 Subject: [PATCH 072/417] precise token and set --- doc/rnnslu.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 98dae465..66531232 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -89,8 +89,8 @@ Recurrent Neural Network Model **Raw input encoding** -Each token in the ATIS vocabulary is associated to an index. Each sentence is a -array of indexes (``int32``). Each set is a list of arrays of indexes. A python +A token corresponds to a word. Each token in the ATIS vocabulary is associated to an index. Each sentence is a +array of indexes (``int32``). Then, each set (train, valid, test) is a list of arrays of indexes. A python dictionnary is defined for mapping the space of indexes to the space of words. >>> sentence From c0ab16e148b710929ad121f05ddd11555818418f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 9 Apr 2014 12:53:18 -0400 Subject: [PATCH 073/417] precise the idxs dimensions --- doc/rnnslu.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 66531232..e3d1554e 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -132,7 +132,7 @@ particular word. In details, we have:: assert win >=1 l = list(l) - lpadded = win/2 * [-1] + l + win/2 * [-1] + lpadded = win//2 * [-1] + l + win//2 * [-1] out = [ lpadded[i:i+win] for i in range(len(l)) ] assert len(out) == len(l) @@ -177,7 +177,7 @@ Using Theano, it gives:: embeddings = theano.shared(0.2 * numpy.random.uniform(-1.0, 1.0, \ (nv+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end - idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence + idxs = T.imatrix() # as many columns as words in the context window and as many lines as words in the sentence x, _ = theano.scan(fn = lambda idx: embeddings[idx].flatten(), sequences = idxs) The x symbolic variable corresponds to a matrix of shape (number of words in the From 4f7d884daccaa52e02eb446922618dd5f8389a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 9 Apr 2014 13:21:17 -0400 Subject: [PATCH 074/417] add reference in the intro and other typos --- doc/intro.txt | 3 +++ doc/rnnslu.txt | 40 ++++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/doc/intro.txt b/doc/intro.txt index 19ab4bc7..bd43434f 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -49,6 +49,9 @@ from energy models: Building towards including the Contractive auto-encoders tutorial, we have the code for now: * `Contractive auto-encoders`_ code - There is some basic doc in the code. +Recurrent neural networks with word embeddings and context window: + * :ref:`Semantic Parsing of Speech using Recurrent Net ` + Energy-based recurrent neural network (RNN-RBM): * :ref:`Modeling and generating sequences of polyphonic music ` diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index e3d1554e..59817ada 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -178,7 +178,7 @@ Using Theano, it gives:: (nv+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end idxs = T.imatrix() # as many columns as words in the context window and as many lines as words in the sentence - x, _ = theano.scan(fn = lambda idx: embeddings[idx].flatten(), sequences = idxs) + x, _ = theano.scan(fn=lambda idx: embeddings[idx].flatten(), sequences=idxs) The x symbolic variable corresponds to a matrix of shape (number of words in the sentences, dimension of the embedding space X context window size). @@ -193,7 +193,7 @@ Let's compile a theano function to do so [-1, 0, 1, 2, 3, 4,-1], [ 0, 1, 2, 3, 4,-1,-1], [ 1, 2, 3, 4,-1,-1,-1]] - >>> f = theano.function( inputs=[idxs], outputs=x) + >>> f = theano.function(inputs=[idxs], outputs=x) >>> f(cx) array([[-0.08088442, 0.08458307, 0.05064092, ..., 0.06876887, -0.06648078, -0.15192257], @@ -250,25 +250,25 @@ It gives the following code:: de :: dimension of the word embeddings cs :: word window context size ''' - self.emb = theano.shared(name='embeddings', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + self.emb = theano.shared(name='embeddings', value=0.2 * numpy.random.uniform(-1.0, 1.0, (ne+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end - self.Wx = theano.shared(name='Wx', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + self.Wx = theano.shared(name='Wx', value=0.2 * numpy.random.uniform(-1.0, 1.0, (de * cs, nh)).astype(theano.config.floatX)) - self.Wh = theano.shared(name='Wh', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + self.Wh = theano.shared(name='Wh', value=0.2 * numpy.random.uniform(-1.0, 1.0, (nh, nh)).astype(theano.config.floatX)) - self.W = theano.shared(name='W', value=0.2 * numpy.random.uniform(-1.0, 1.0,\ + self.W = theano.shared(name='W', value=0.2 * numpy.random.uniform(-1.0, 1.0, (nh, nc)).astype(theano.config.floatX)) self.bh = theano.shared(name='bh', value=numpy.zeros(nh, dtype=theano.config.floatX)) self.b = theano.shared(name='b', value=numpy.zeros(nc, dtype=theano.config.floatX)) self.h0 = theano.shared(name='h0', value=numpy.zeros(nh, dtype=theano.config.floatX)) # bundle - self.params = [ self.emb, self.Wx, self.Wh, self.W, self.bh, self.b, self.h0 ] + self.params = [self.emb, self.Wx, self.Wh, self.W, self.bh, self.b, self.h0] Then we integrate the way to build the input from the embedding matrix:: idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence - x, _ = theano.scan(fn = lambda idx: self.emb[idx].flatten(), sequences = idxs) + x, _ = theano.scan(fn=lambda idx: self.emb[idx].flatten(), sequences=idxs) y = T.ivector('y') # label We use the scan operator to construct the recursion, works like a charm:: @@ -278,11 +278,11 @@ We use the scan operator to construct the recursion, works like a charm:: s_t = T.nnet.softmax(T.dot(h_t, self.W) + self.b) return [h_t, s_t] - [h, s], _ = theano.scan(fn=recurrence, \ - sequences=x, outputs_info=[self.h0, None], \ + [h, s], _ = theano.scan(fn=recurrence, + sequences=x, outputs_info=[self.h0, None], n_steps=x.shape[0]) - p_y_given_x_sentence = s[:,0,:] + p_y_given_x_sentence = s[:, 0, :] y_pred = T.argmax(p_y_given_x_sentence, axis=1) Theano will then compute all the gradients automatically to maximize the log-likelihood:: @@ -290,21 +290,21 @@ Theano will then compute all the gradients automatically to maximize the log-lik lr = T.scalar('lr') nll = -T.mean(T.log(p_y_given_x_sentence)[T.arange(x.shape[0]),y]) gradients = T.grad( nll, self.params ) - updates = OrderedDict(( p, p-lr*g ) for p, g in zip( self.params , gradients)) + updates = OrderedDict((p, p - lr*g) for p, g in zip(self.params, gradients)) Next compile those functions:: self.classify = theano.function(inputs=[idxs], outputs=y_pred) - self.train = theano.function( inputs = [idxs, y, lr], - outputs = nll, - updates = updates ) + self.train = theano.function(inputs=[idxs, y, lr], + outputs=nll, + updates=updates) We keep the word embeddings on the unit sphere by normalizing them after each update:: - self.normalize = theano.function( inputs = [], - updates = {self.emb:\ - self.emb/T.sqrt((self.emb**2).sum(axis=1)).dimshuffle(0,'x')}) + self.normalize = theano.function(inputs=[], + updates = {self.emb: + self.emb / T.sqrt((self.emb**2).sum(axis=1)).dimshuffle(0, 'x')}) And that's it! @@ -358,7 +358,7 @@ Results **Timing** Running experiments on ATIS using this `repository `_ -will run one epoch in less than 40 seconds on bart1 processor using less than 200 Mo of RAM:: +will run one epoch in less than 40 seconds on i7 CPU 950 @ 3.07GHz using less than 200 Mo of RAM:: [learning] epoch 0 >> 100.00% completed in 34.48 (sec) << @@ -378,7 +378,7 @@ After a few epochs, you obtain decent performance **94.48 % of F1 score**.:: **Word Embedding Nearest Neighbors** -We can check the k-nearest neighbors of the thus learned embeddings. L2 and +We can check the k-nearest neighbors of the learned embeddings. L2 and cosine distance gave the same results so we plot them for the cosine distance. +------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+------------------------------+ From ed02dd9a2760b2ba863fb527e73e1807f92660ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Fri, 11 Apr 2014 19:28:23 -0400 Subject: [PATCH 075/417] idxs fix rather than scan --- doc/rnnslu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 59817ada..a243ac31 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -268,7 +268,7 @@ It gives the following code:: Then we integrate the way to build the input from the embedding matrix:: idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence - x, _ = theano.scan(fn=lambda idx: self.emb[idx].flatten(), sequences=idxs) + x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) y = T.ivector('y') # label We use the scan operator to construct the recursion, works like a charm:: From 7b7e603f8e0021aef0c297967554df9fb097eb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Fri, 11 Apr 2014 20:38:43 -0400 Subject: [PATCH 076/417] add script --- code/rnnslu.py | 356 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 code/rnnslu.py diff --git a/code/rnnslu.py b/code/rnnslu.py new file mode 100644 index 00000000..79c57923 --- /dev/null +++ b/code/rnnslu.py @@ -0,0 +1,356 @@ +import numpy +import time +import sys +import subprocess +import os +import random +import copy +import gzip +import cPickle + +from collections import OrderedDict + +import theano +from theano import tensor as T + +PREFIX = os.getenv('ATISDATA', '') + + +# utils functions +def shuffle(lol, seed): + ''' + lol :: list of list as input + seed :: seed the shuffling + + shuffle inplace each list in the same order + ''' + for l in lol: + random.seed(seed) + random.shuffle(l) + + +def contextwin(l, win): + ''' + win :: int corresponding to the size of the window + given a list of indexes composing a sentence + it will return a list of list of indexes corresponding + to context windows surrounding each word in the sentence + ''' + assert (win % 2) == 1 + assert win >= 1 + l = list(l) + + lpadded = win//2 * [-1] + l + win//2 * [-1] + out = [lpadded[i:i+win] for i in range(len(l))] + + assert len(out) == len(l) + return out + + +# data loading functions +def atisfold(fold): + assert fold in range(5) + filename = os.path.join(PREFIX, 'atis.fold'+str(fold)+'.pkl.gz') + f = gzip.open(filename, 'rb') + train_set, valid_set, test_set, dicts = cPickle.load(f) + return train_set, valid_set, test_set, dicts + + +# metrics function using conlleval.pl +def conlleval(p, g, w, filename): + ''' + INPUT: + p :: predictions + g :: groundtruth + w :: corresponding words + + OUTPUT: + filename :: name of the file where the predictions + are written. it will be the input of conlleval.pl script + for computing the performance in terms of precision + recall and f1 score + ''' + out = '' + for sl, sp, sw in zip(g, p, w): + out += 'BOS O O\n' + for wl, wp, w in zip(sl, sp, sw): + out += w + ' ' + wl + ' ' + wp + '\n' + out += 'EOS O O\n\n' + + f = open(filename, 'w') + f.writelines(out) + f.close() + + return get_perf(filename) + + +def get_perf(filename): + ''' run conlleval.pl perl script to obtain + precision/recall and F1 score ''' + _conlleval = PREFIX + 'conlleval.pl' + if not os.path.isfile(_conlleval): + # url = 'http://www-etud.iro.umontreal.ca/ + # ~mesnilgr/atis/conlleval.pl' + download(url) + chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions + + proc = subprocess.Popen(["perl", + _conlleval], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + + stdout, _ = proc.communicate(''.join(open(filename).readlines())) + for line in stdout.split('\n'): + if 'accuracy' in line: + out = line.split() + break + + precision = float(out[6][:-2]) + recall = float(out[8][:-2]) + f1score = float(out[10]) + + return {'p': precision, 'r': recall, 'f1': f1score} + + +# actual model +class basemodel(object): + ''' load/save structure ''' + + def save(self, folder): + for param in self.params: + numpy.save(os.path.join(folder, + param.name + '.npy'), param.get_value()) + + def load(self, folder): + for param in self.params: + param.set_value(numpy.load(os.path.join(folder, + param.name + '.npy'))) + + +class model(basemodel): + ''' elman neural net model ''' + def __init__(self, nh, nc, ne, de, cs): + ''' + nh :: dimension of the hidden layer + nc :: number of classes + ne :: number of word embeddings in the vocabulary + de :: dimension of the word embeddings + cs :: word window context size + ''' + # parameters of the model + self.emb = theano.shared(name='embeddings', + value=0.2 * numpy.random.uniform(-1.0, 1.0, + (ne+1, de)) + # add one for padding at the end + .astype(theano.config.floatX)) + self.wx = theano.shared(name='wx', + value=0.2 * numpy.random.uniform(-1.0, 1.0, + (de * cs, nh)) + .astype(theano.config.floatX)) + self.wh = theano.shared(name='wh', + value=0.2 * numpy.random.uniform(-1.0, 1.0, + (nh, nh)) + .astype(theano.config.floatX)) + self.w = theano.shared(name='w', + value=0.2 * numpy.random.uniform(-1.0, 1.0, + (nh, nc)) + .astype(theano.config.floatX)) + self.bh = theano.shared(name='bh', + value=numpy.zeros(nh, + dtype=theano.config.floatX)) + self.b = theano.shared(name='b', + value=numpy.zeros(nc, + dtype=theano.config.floatX)) + self.h0 = theano.shared(name='h0', + value=numpy.zeros(nh, + dtype=theano.config.floatX)) + + # bundle + self.params = [self.emb, self.wx, self.wh, self.w, + self.bh, self.b, self.h0] + # as many columns as context window size + # as many lines as words in the sentence + idxs = T.imatrix() + x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) + y_sentence = T.ivector('y_sentence') # labels + + def recurrence(x_t, h_tm1): + h_t = T.nnet.sigmoid(T.dot(x_t, self.wx) + + T.dot(h_tm1, self.wh) + self.bh) + s_t = T.nnet.softmax(T.dot(h_t, self.w) + self.b) + return [h_t, s_t] + + [h, s], _ = theano.scan(fn=recurrence, + sequences=x, + outputs_info=[self.h0, None], + n_steps=x.shape[0]) + + p_y_given_x_sentence = s[:, 0, :] + y_pred = T.argmax(p_y_given_x_sentence, axis=1) + + # cost and gradients and learning rate + lr = T.scalar('lr') + + sentence_nll = -T.mean(T.log(p_y_given_x_sentence) + [T.arange(x.shape[0]), y_sentence]) + sentence_gradients = T.grad(sentence_nll, self.params) + sentence_updates = OrderedDict((p, p - lr*g) + for p, g in + zip(self.params, sentence_gradients)) + + # theano functions to compile + self.classify = theano.function(inputs=[idxs], outputs=y_pred) + self.sentence_train = theano.function(inputs=[idxs, y_sentence, lr], + outputs=sentence_nll, + updates=sentence_updates) + self.normalize = theano.function(inputs=[], + updates={self.emb: + self.emb / + T.sqrt((self.emb**2) + .sum(axis=1)) + .dimshuffle(0, 'x')}) + + def train(self, x, y, window_size, learning_rate): + + cwords = contextwin(x, window_size) + words = map(lambda x: numpy.asarray(x).astype('int32'), cwords) + labels = y + + self.sentence_train(words, labels, learning_rate) + self.normalize() + + +def main(param, sync=None): + + folder = os.path.basename(__file__).split('.')[0] + if not os.path.exists(folder): + os.mkdir(folder) + + # load the dataset + train_set, valid_set, test_set, dic = atisfold(param['fold']) + + idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems()) + idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems()) + + train_lex, train_ne, train_y = train_set + valid_lex, valid_ne, valid_y = valid_set + test_lex, test_ne, test_y = test_set + + vocsize = len(set(reduce(lambda x, y: list(x) + list(y), + train_lex + valid_lex + test_lex))) + nclasses = len(set(reduce(lambda x, y: list(x)+list(y), + train_y + test_y + valid_y))) + nsentences = len(train_lex) + + groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y] + words_valid = [map(lambda x: idx2word[x], w) for w in valid_lex] + groundtruth_test = [map(lambda x: idx2label[x], y) for y in test_y] + words_test = [map(lambda x: idx2word[x], w) for w in test_lex] + + # instanciate the model + numpy.random.seed(param['seed']) + random.seed(param['seed']) + + rnn = model(nh=param['nhidden'], + nc=nclasses, + ne=vocsize, + de=param['emb_dimension'], + cs=param['win']) + + # train with early stopping on validation set + best_f1 = -numpy.inf + param['clr'] = param['lr'] + for e in xrange(param['nepochs']): + + # shuffle + shuffle([train_lex, train_ne, train_y], param['seed']) + + param['ce'] = e + tic = time.time() + + for i, (x, y) in enumerate(zip(train_lex, train_y)): + rnn.train(x, y, param['win'], param['clr']) + + # evaluation // back into the real world : idx -> words + predictions_test = [map(lambda x: idx2label[x], + rnn.classify(numpy.asarray( + contextwin(x, param['win'])).astype('int32'))) + for x in test_lex] + predictions_valid = [map(lambda x: idx2label[x], + rnn.classify(numpy.asarray( + contextwin(x, param['win'])).astype('int32'))) + for x in valid_lex] + + # evaluation // compute the accuracy using conlleval.pl + res_test = conlleval(predictions_test, + groundtruth_test, + words_test, + folder + '/current.test.txt') + res_valid = conlleval(predictions_valid, + groundtruth_valid, + words_valid, + folder + '/current.valid.txt') + + if res_valid['f1'] > best_f1: + + if sync is not None: + sync() + if param['savemodel']: + rnn.save(folder) + + best_rnn = copy.deepcopy(rnn) + best_f1 = res_valid['f1'] + + if param['verbose']: + print('NEW BEST: epoch', e, + 'valid F1', res_valid['f1'], + 'best test F1', res_test['f1']) + + param['vf1'], param['tf1'] = res_valid['f1'], res_test['f1'] + param['vp'], param['tp'] = res_valid['p'], res_test['p'] + param['vr'], param['tr'] = res_valid['r'], res_test['r'] + param['be'] = e + + subprocess.call(['mv', folder + '/current.test.txt', + folder + '/best.test.txt']) + subprocess.call(['mv', folder + '/current.valid.txt', + folder + '/best.valid.txt']) + else: + if param['verbose']: + print '' + + # learning rate decay if no improvement in 10 epochs + if param['decay'] and abs(param['be']-param['ce']) >= 10: + param['clr'] *= 0.5 + rnn = best_rnn + + if param['clr'] < 1e-5: + break + + print('BEST RESULT: epoch', param['be'], + 'valid F1', param['vf1'], + 'best test F1', param['tf1'], + 'with the model', folder) + +if __name__ == '__main__': + + # best model + s = {'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 60, + 'savemodel': True} + + main(s) + print s From 9d856f7a156cd7dc6792ccdefb0a3ef7a49a2c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Fri, 11 Apr 2014 20:46:07 -0400 Subject: [PATCH 077/417] added test for 1 epoch --- code/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 30025c00..a97af7a2 100644 --- a/code/test.py +++ b/code/test.py @@ -11,6 +11,11 @@ import rbm import rnnrbm import SdA +import rnnslu + + +def test_rnnslu(): + rnnslu.test(1) def test_logistic_sgd(): @@ -55,7 +60,6 @@ def test_rbm(): def test_rnnrbm(): rnnrbm.test_rnnrbm(num_epochs=1) - def speed(): """ This fonction modify the configuration theano and don't restore it! From 21664bf96c54f4d3bc8b082e17b2b707a39cdc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 15 Apr 2014 15:09:10 -0400 Subject: [PATCH 078/417] fix typo stuff --- code/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index a97af7a2..bf6a8fd2 100644 --- a/code/test.py +++ b/code/test.py @@ -15,7 +15,7 @@ def test_rnnslu(): - rnnslu.test(1) + rnnslu.test_rnnslu(n_epochs=1) def test_logistic_sgd(): @@ -60,6 +60,7 @@ def test_rbm(): def test_rnnrbm(): rnnrbm.test_rnnrbm(num_epochs=1) + def speed(): """ This fonction modify the configuration theano and don't restore it! From b7c8d0443b77b8310a349219ee804cde01a72dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 15 Apr 2014 15:09:30 -0400 Subject: [PATCH 079/417] fixes according to reviews --- code/rnnslu.py | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 79c57923..e8182846 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -112,22 +112,7 @@ def get_perf(filename): return {'p': precision, 'r': recall, 'f1': f1score} -# actual model -class basemodel(object): - ''' load/save structure ''' - - def save(self, folder): - for param in self.params: - numpy.save(os.path.join(folder, - param.name + '.npy'), param.get_value()) - - def load(self, folder): - for param in self.params: - param.set_value(numpy.load(os.path.join(folder, - param.name + '.npy'))) - - -class model(basemodel): +class RNNSLU(object): ''' elman neural net model ''' def __init__(self, nh, nc, ne, de, cs): ''' @@ -219,8 +204,19 @@ def train(self, x, y, window_size, learning_rate): self.sentence_train(words, labels, learning_rate) self.normalize() + def save(self, folder): + for param in self.params: + numpy.save(os.path.join(folder, + param.name + '.npy'), param.get_value()) + + def load(self, folder): + for param in self.params: + param.set_value(numpy.load(os.path.join(folder, + param.name + '.npy'))) -def main(param, sync=None): + + +def main(param): folder = os.path.basename(__file__).split('.')[0] if not os.path.exists(folder): @@ -251,11 +247,11 @@ def main(param, sync=None): numpy.random.seed(param['seed']) random.seed(param['seed']) - rnn = model(nh=param['nhidden'], - nc=nclasses, - ne=vocsize, - de=param['emb_dimension'], - cs=param['win']) + rnn = RNNSLU(nh=param['nhidden'], + nc=nclasses, + ne=vocsize, + de=param['emb_dimension'], + cs=param['win']) # train with early stopping on validation set best_f1 = -numpy.inf @@ -293,8 +289,6 @@ def main(param, sync=None): if res_valid['f1'] > best_f1: - if sync is not None: - sync() if param['savemodel']: rnn.save(folder) @@ -332,8 +326,7 @@ def main(param, sync=None): 'best test F1', param['tf1'], 'with the model', folder) -if __name__ == '__main__': - +def test_rnnslu(n_epochs): # best model s = {'fold': 3, # 5 folds 0,1,2,3,4 @@ -349,7 +342,8 @@ def main(param, sync=None): 'seed': 345, 'emb_dimension': 50, # dimension of word embedding - 'nepochs': 60, + 'nepochs': n_epochs, + # 60 is recommended 'savemodel': True} main(s) From f9c425b1033f582beda9374ba6208738796e9c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 15 Apr 2014 15:11:40 -0400 Subject: [PATCH 080/417] review fix --- doc/rnnslu.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index a243ac31..16d4f7ff 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -172,13 +172,14 @@ Using Theano, it gives:: # nv :: size of our vocabulary # de :: dimension of the embedding space - nv, de = 1000, 50 + # cs :: context window size + nv, de, cs = 1000, 50, 5 embeddings = theano.shared(0.2 * numpy.random.uniform(-1.0, 1.0, \ (nv+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end idxs = T.imatrix() # as many columns as words in the context window and as many lines as words in the sentence - x, _ = theano.scan(fn=lambda idx: embeddings[idx].flatten(), sequences=idxs) + x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) The x symbolic variable corresponds to a matrix of shape (number of words in the sentences, dimension of the embedding space X context window size). From e1c18399716431ec8d2cce4ab17fecfc8f8c6291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 15 Apr 2014 16:12:08 -0400 Subject: [PATCH 081/417] new fix --- code/rnnslu.py | 3 ++- code/test.py | 21 ++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index e8182846..6f1159d7 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -326,7 +326,8 @@ def main(param): 'best test F1', param['tf1'], 'with the model', folder) -def test_rnnslu(n_epochs): + +if __name__ == '__main__': # best model s = {'fold': 3, # 5 folds 0,1,2,3,4 diff --git a/code/test.py b/code/test.py index bf6a8fd2..4cad74b5 100644 --- a/code/test.py +++ b/code/test.py @@ -15,7 +15,26 @@ def test_rnnslu(): - rnnslu.test_rnnslu(n_epochs=1) + # best model + s = {'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 1, + # 60 is recommended + 'savemodel': True} + + rnnslu.main(s) def test_logistic_sgd(): From 14e9583a31d03aea54d478138da277166707829b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 16 Apr 2014 20:40:30 -0400 Subject: [PATCH 082/417] changing extension --- doc/rnnslu.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 16d4f7ff..803e1275 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -320,6 +320,7 @@ It's not trivial to compute those metrics due to the `Inside Outside Beginning (IOB) `_ representation i.e. a prediction is considered correct if the word-beginnin **and** the word-inside **and** the word-outside predictions are **all** correct. +Note that the extension is `txt` and you will have to cahnge it to `pl`. Training ++++++++ From 184d938416178ea42f92e8be05526d9e47ec4373 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 16 Apr 2014 13:35:57 -0400 Subject: [PATCH 083/417] fix crash --- code/rnnslu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 6f1159d7..4538a53a 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -343,7 +343,7 @@ def main(param): 'seed': 345, 'emb_dimension': 50, # dimension of word embedding - 'nepochs': n_epochs, + 'nepochs': 60, # 60 is recommended 'savemodel': True} From 475866e059525dcb5686b96820734a5f822dcf85 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 16 Apr 2014 13:36:45 -0400 Subject: [PATCH 084/417] Don't save the model by default. --- code/rnnslu.py | 2 +- code/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 4538a53a..18e88c37 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -345,7 +345,7 @@ def main(param): # dimension of word embedding 'nepochs': 60, # 60 is recommended - 'savemodel': True} + 'savemodel': False} main(s) print s diff --git a/code/test.py b/code/test.py index 4cad74b5..1ddea3f1 100644 --- a/code/test.py +++ b/code/test.py @@ -32,7 +32,7 @@ def test_rnnslu(): # dimension of word embedding 'nepochs': 1, # 60 is recommended - 'savemodel': True} + 'savemodel': False} rnnslu.main(s) From f2d83201945cc946eab0571b95b0375758d424e1 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 16 Apr 2014 13:37:03 -0400 Subject: [PATCH 085/417] Add speed test for rnnslu. --- code/test.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/code/test.py b/code/test.py index 1ddea3f1..59cf0c11 100644 --- a/code/test.py +++ b/code/test.py @@ -86,7 +86,7 @@ def speed(): """ algo = ['logistic_sgd', 'logistic_cg', 'mlp', 'convolutional_mlp', - 'dA', 'SdA', 'DBN', 'rbm', 'rnnrbm'] + 'dA', 'SdA', 'DBN', 'rbm', 'rnnrbm', 'rnnslu'] to_exec = [True] * len(algo) # to_exec = [False] * len(algo) # to_exec[-1] = True @@ -101,9 +101,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.8, 22.5, 76.1, 73.7, 116.4, - 346.9, 381.9, 558.1, 186.3]) + 346.9, 381.9, 558.1, 186.3, 50.8]) expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, - 191.2, 226.8, 432.8, 176.2]) + 191.2, 226.8, 432.8, 176.2, 36.9]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -124,8 +124,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 21.5, - 11.8, 47.9, 290.1, 315.4]) - + 11.8, 47.9, 290.1, 315.4, 72.4]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) @@ -164,6 +163,24 @@ def do_tests(): time_test(m, l, 7, rbm.test_rbm, training_epochs=1, batch_size=300, n_chains=1, n_samples=1, output_folder='tmp_rbm_plots') time_test(m, l, 8, rnnrbm.test_rnnrbm, num_epochs=1) + s = {'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 1, + # 60 is recommended + 'savemodel': False} + time_test(m, l, 9, rnnslu.main, param=s) return numpy.asarray(l) #test in float64 in FAST_RUN mode on the cpu From 0c65cfc558ab11638b20f63ffad037a6afd90c16 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 16 Apr 2014 16:39:44 -0400 Subject: [PATCH 086/417] Add misssing import. --- doc/lenet.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/lenet.txt b/doc/lenet.txt index 7971cc7e..56636d9f 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -255,6 +255,7 @@ Let's have a little bit of fun with this... .. code-block:: python + import numpy import pylab from PIL import Image From 7570c3eddb7010a04b99a0b472dc36751a9601dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 16 Apr 2014 20:55:57 -0400 Subject: [PATCH 087/417] add download of evaluation perl script --- code/rnnslu.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 18e88c37..2bb885a1 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -84,13 +84,22 @@ def conlleval(p, g, w, filename): return get_perf(filename) +def download(origin): + ''' + download the corresponding atis file + from http://www-etud.iro.umontreal.ca/~mesnilgr/atis/ + ''' + print 'Downloading data from %s' % origin + name = origin.split('/')[-1] + urllib.urlretrieve(origin, name) + + def get_perf(filename): ''' run conlleval.pl perl script to obtain precision/recall and F1 score ''' _conlleval = PREFIX + 'conlleval.pl' if not os.path.isfile(_conlleval): - # url = 'http://www-etud.iro.umontreal.ca/ - # ~mesnilgr/atis/conlleval.pl' + url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl' download(url) chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions From 1f2bd27721de9bd74f573ed82b5dad4d2317080f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 16 Apr 2014 21:04:01 -0400 Subject: [PATCH 088/417] add suggested references from MSR people --- doc/rnnslu.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 803e1275..38e5daa4 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -25,6 +25,10 @@ If you use this tutorial, cite the following papers: * `[pdf] `_ Grégoire Mesnil, Xiaodong He, Li Deng and Yoshua Bengio. Investigation of Recurrent-Neural-Network Architectures and Learning Methods for Spoken Language Understanding. Interspeech, 2013. +* `[pdf] `_ Gokhan Tur, Dilek Hakkani-Tur and Larry Heck. What is left to be understood in ATIS? + +* `[pdf] `_ Christian Raymond and Giuseppe Riccardi. Generative and discriminative algorithms for spoken language understanding. Interspeech, 2007. + * `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. * `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. From 23e72bed2006540f1d272e07560e00bc2c966c71 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 6 Jun 2014 15:31:44 -0400 Subject: [PATCH 089/417] Add missing urllib import and order import as pep8. --- code/rnnslu.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 2bb885a1..699722d5 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -1,14 +1,15 @@ -import numpy -import time -import sys -import subprocess -import os -import random +from collections import OrderedDict import copy -import gzip import cPickle +import gzip +import os +import urllib +import random +import subprocess +import sys +import time -from collections import OrderedDict +import numpy import theano from theano import tensor as T From c99271343b60b2cf8b4c2e63599203dbc48a6c1a Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 6 Jun 2014 15:32:19 -0400 Subject: [PATCH 090/417] Add download of dataset. --- data/download.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data/download.sh b/data/download.sh index 2f1cf77d..e2a837b9 100755 --- a/data/download.sh +++ b/data/download.sh @@ -17,3 +17,8 @@ $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" +$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz +$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold1.pkl.gz +$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold2.pkl.gz +$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold3.pkl.gz +$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold4.pkl.gz From a262f810fe04a6cef96840a2f16e4585d243a5d9 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 6 Jun 2014 15:41:25 -0400 Subject: [PATCH 091/417] small fix --- code/rnnslu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 699722d5..28aa478e 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -5,6 +5,7 @@ import os import urllib import random +import stat import subprocess import sys import time @@ -102,7 +103,7 @@ def get_perf(filename): if not os.path.isfile(_conlleval): url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl' download(url) - chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions + os.chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions proc = subprocess.Popen(["perl", _conlleval], From af03be54f31345a39e8c4399886417697890a9b5 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 6 Jun 2014 16:40:40 -0400 Subject: [PATCH 092/417] some updates. --- code/rnnslu.py | 4 ++-- doc/rnnslu.txt | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 28aa478e..affd60c6 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -15,7 +15,7 @@ import theano from theano import tensor as T -PREFIX = os.getenv('ATISDATA', '') +PREFIX = os.getenv('ATISDATA', 'data') # utils functions @@ -99,7 +99,7 @@ def download(origin): def get_perf(filename): ''' run conlleval.pl perl script to obtain precision/recall and F1 score ''' - _conlleval = PREFIX + 'conlleval.pl' + _conlleval = 'conlleval.pl' if not os.path.isfile(_conlleval): url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl' download(url) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 38e5daa4..73411518 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -358,8 +358,17 @@ The following intervals can give you some starting point: * number of hidden units : random value from {100,200} * embedding dimension : random value from {50,100} -Results -+++++++ +Running the Code +++++++++++++++++ + +The user can then run the code by calling: + +.. code-block:: bash + + python code/rnnslu.py + +UPDATE THE OUTPUT TO THAT SCRIPT + **Timing** From 6568c7380b3951d6959885b8078300b184a9f4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 23 Sep 2014 18:44:51 -0400 Subject: [PATCH 093/417] debuging --- code/rnnslu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index affd60c6..6c35bb86 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -297,7 +297,7 @@ def main(param): groundtruth_valid, words_valid, folder + '/current.valid.txt') - + if res_valid['f1'] > best_f1: if param['savemodel']: @@ -358,5 +358,5 @@ def main(param): # 60 is recommended 'savemodel': False} - main(s) print s + main(s) From 86162c6b120aa4bb98b6c329cdf6bc3fe6e4bf1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Tue, 23 Sep 2014 19:06:56 -0400 Subject: [PATCH 094/417] add timing output --- code/rnnslu.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 6c35bb86..9f0ef0bb 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -277,7 +277,9 @@ def main(param): for i, (x, y) in enumerate(zip(train_lex, train_y)): rnn.train(x, y, param['win'], param['clr']) - + print '[learning] epoch %i >> %2.2f%%'%(e,(i+1)*100./nsentences),'completed in %.2f (sec) <<\r'%(time.time()-tic), + sys.stdout.flush() + # evaluation // back into the real world : idx -> words predictions_test = [map(lambda x: idx2label[x], rnn.classify(numpy.asarray( From ff9b62db5524b9652eacc1469743eb1b9c37ef13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Wed, 24 Sep 2014 16:54:27 -0400 Subject: [PATCH 095/417] add output to the doc --- doc/rnnslu.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 73411518..c7e83470 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -367,8 +367,11 @@ The user can then run the code by calling: python code/rnnslu.py -UPDATE THE OUTPUT TO THAT SCRIPT - + ('NEW BEST: epoch', 25, 'valid F1', 96.84, 'best test F1', 93.79) + [learning] epoch 26 >> 100.00% completed in 28.76 (sec) << + [learning] epoch 27 >> 100.00% completed in 28.76 (sec) << + ... + ('BEST RESULT: epoch', 57, 'valid F1', 97.23, 'best test F1', 94.2, 'with the model', 'rnnslu') **Timing** From 74aba5c75fbf0d9ee683053cdd6a78eb39e865f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 17:56:28 -0500 Subject: [PATCH 096/417] convert bold subtitles to subsections --- doc/rnnslu.txt | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index c7e83470..4a154cf9 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -1,3 +1,5 @@ +.. _rnnslu: + Recurrent Neural Networks with Word Embeddings ********************************************** @@ -15,11 +17,13 @@ in order to perform Semantic Parsing / Slot-Filling (Spoken Language Understandi Code - Citations - Contact ++++++++++++++++++++++++++ -**Code** +Code +==== Directly running experiments is also possible using this `github repository `_. -**Papers** +Papers +====== If you use this tutorial, cite the following papers: @@ -35,7 +39,8 @@ If you use this tutorial, cite the following papers: Thank you! -**Contact** +Contact +======= Please email to `Grégoire Mesnil `_ for any problem report or feedback. We will be glad to hear from you. @@ -91,7 +96,8 @@ measure the performance of our models. Recurrent Neural Network Model ++++++++++++++++++++++++++++++ -**Raw input encoding** +Raw input encoding +================== A token corresponds to a word. Each token in the ATIS vocabulary is associated to an index. Each sentence is a array of indexes (``int32``). Then, each set (train, valid, test) is a list of arrays of indexes. A python @@ -115,7 +121,8 @@ Same thing for labels corresponding to this particular sentence. 'O', 'B-arrive_time.time_relative', 'B-arrive_time.time', 'I-arrive_time.time', 'I-arrive_time.time'] -**Context window** +Context window +============== Given a sentence i.e. an array of indexes, and a window size i.e. 1,3,5,..., we need to convert each word in the sentence to a context window surrounding this @@ -165,7 +172,8 @@ Here is a sample: To summarize, we started with an array of indexes and ended with a matrix of indexes. Each line corresponds to the context window surrounding this word. -**Word embeddings** +Word embeddings +================= Once we have the sentence converted to context windows i.e. a matrix of indexes, we have to associate these indexes to the embeddings (real-valued vector associated to each word). @@ -218,7 +226,8 @@ We now have a sequence (of length 5 which is corresponds to the length of the sentence) of **context window word embeddings** which is easy to feed to a simple recurrent neural network to iterate with. -**Elman recurrent neural network** +Elman recurrent neural network +============================== The followin (Elman) recurrent neural network (E-RNN) takes as input the current input (time ``t``) and the previous hiddent state (time ``t-1``). Then it iterates. @@ -329,7 +338,8 @@ Note that the extension is `txt` and you will have to cahnge it to `pl`. Training ++++++++ -**Updates** +Updates +======= For stochastic gradient descent (SGD) update, we consider the whole sentence as a mini-batch and perform one update per sentence. It is possible to perform a pure SGD (contrary to mini-batch) @@ -337,14 +347,16 @@ where the update is done on only one single word at a time. After each iteration/update, we normalize the word embeddings to keep them on a unit sphere. -**Stopping Criterion** +Stopping Criterion +================== Early-stopping on a validation set is our regularization technique: the training is run for a given number of epochs (a single pass through the whole dataset) and keep the best model along with respect to the F1 score computed on the validation set after each epoch. -**Hyper-Parameter Selection** +Hyper-Parameter Selection +========================= Although there is interesting research/`code `_ on the topic of automatic @@ -373,7 +385,8 @@ The user can then run the code by calling: ... ('BEST RESULT: epoch', 57, 'valid F1', 97.23, 'best test F1', 94.2, 'with the model', 'rnnslu') -**Timing** +Timing +====== Running experiments on ATIS using this `repository `_ will run one epoch in less than 40 seconds on i7 CPU 950 @ 3.07GHz using less than 200 Mo of RAM:: @@ -394,7 +407,8 @@ After a few epochs, you obtain decent performance **94.48 % of F1 score**.:: [learning] epoch 44 >> 100.00% completed in 35.31 (sec) << [...] -**Word Embedding Nearest Neighbors** +Word Embedding Nearest Neighbors +================================ We can check the k-nearest neighbors of the learned embeddings. L2 and cosine distance gave the same results so we plot them for the cosine distance. From ada4162aa7ea8932692f47f119abb9ecc9522069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:19:17 -0500 Subject: [PATCH 097/417] remove dictionnary duplication --- code/rnnslu.py | 45 ++++++++++++++++++++++----------------------- code/test.py | 21 +-------------------- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 9f0ef0bb..b8f2ceb2 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -227,8 +227,27 @@ def load(self, folder): -def main(param): - +def main(param=None): + if not param: + param = {'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 60, + # 60 is recommended + 'savemodel': False} + print param + folder = os.path.basename(__file__).split('.')[0] if not os.path.exists(folder): os.mkdir(folder) @@ -341,24 +360,4 @@ def main(param): if __name__ == '__main__': - # best model - s = {'fold': 3, - # 5 folds 0,1,2,3,4 - 'data': 'atis', - 'lr': 0.0970806646812754, - 'verbose': 1, - 'decay': True, - # decay on the learning rate if improvement stops - 'win': 7, - # number of words in the context window - 'nhidden': 200, - # number of hidden units - 'seed': 345, - 'emb_dimension': 50, - # dimension of word embedding - 'nepochs': 60, - # 60 is recommended - 'savemodel': False} - - print s - main(s) + main() diff --git a/code/test.py b/code/test.py index 59cf0c11..b366dbbc 100644 --- a/code/test.py +++ b/code/test.py @@ -15,26 +15,7 @@ def test_rnnslu(): - # best model - s = {'fold': 3, - # 5 folds 0,1,2,3,4 - 'data': 'atis', - 'lr': 0.0970806646812754, - 'verbose': 1, - 'decay': True, - # decay on the learning rate if improvement stops - 'win': 7, - # number of words in the context window - 'nhidden': 200, - # number of hidden units - 'seed': 345, - 'emb_dimension': 50, - # dimension of word embedding - 'nepochs': 1, - # 60 is recommended - 'savemodel': False} - - rnnslu.main(s) + rnnslu.main() def test_logistic_sgd(): From f920e6b2b096026b81caef268ed58e1dd9f12c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:22:29 -0500 Subject: [PATCH 098/417] fix discrepancy with symbolic variable --- doc/rnnslu.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 4a154cf9..36d43d9b 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -198,16 +198,16 @@ sentences, dimension of the embedding space X context window size). Let's compile a theano function to do so - >>> x + >>> sample array([0, 1, 2, 3, 4], dtype=int32) - >>> cx = contextwin(x, 7) + >>> csample = contextwin(sample, 7) [[-1, -1, -1, 0, 1, 2, 3], [-1, -1, 0, 1, 2, 3, 4], [-1, 0, 1, 2, 3, 4,-1], [ 0, 1, 2, 3, 4,-1,-1], [ 1, 2, 3, 4,-1,-1,-1]] >>> f = theano.function(inputs=[idxs], outputs=x) - >>> f(cx) + >>> f(csample) array([[-0.08088442, 0.08458307, 0.05064092, ..., 0.06876887, -0.06648078, -0.15192257], [-0.08088442, 0.08458307, 0.05064092, ..., 0.11192625, @@ -218,7 +218,7 @@ Let's compile a theano function to do so 0.10804889, 0.1247109 ], [ 0.18738101, 0.14727569, -0.069544 , ..., -0.00937143, 0.10804889, 0.1247109 ]], dtype=float32) - >>> f(cx).shape + >>> f(csample).shape (5, 350) From cf441d2d56eeae52b40734418ffe760aee08aeb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:23:41 -0500 Subject: [PATCH 099/417] add word --- doc/rnnslu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 36d43d9b..3361385d 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -233,7 +233,7 @@ The followin (Elman) recurrent neural network (E-RNN) takes as input the current (time ``t``) and the previous hiddent state (time ``t-1``). Then it iterates. In the previous section, we processed the input to fit this -sequential/temporal. It consists in a matrix where the row ``0`` corresponds to +sequential/temporal structure. It consists in a matrix where the row ``0`` corresponds to the time step ``t=0``, the row ``1`` corresponds to the time step ``t=1``, etc. The **parameters** of the E-RNN to be learned are: From cd775c31ba5afd8604550a8cb719198266ee0508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:24:48 -0500 Subject: [PATCH 100/417] fix class name --- doc/rnnslu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 3361385d..b2bd9373 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -254,7 +254,7 @@ The **hyperparameters** define the whole architecture: It gives the following code:: - class ERNN(object): + class RNNSLU(object): def __init__(self, nh, nc, ne, de, cs): ''' From 723dedb8d045a10c209546e54bb95cfed41d0d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:25:29 -0500 Subject: [PATCH 101/417] typo --- doc/rnnslu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index b2bd9373..68f9b3a3 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -333,7 +333,7 @@ It's not trivial to compute those metrics due to the `Inside Outside Beginning (IOB) `_ representation i.e. a prediction is considered correct if the word-beginnin **and** the word-inside **and** the word-outside predictions are **all** correct. -Note that the extension is `txt` and you will have to cahnge it to `pl`. +Note that the extension is `txt` and you will have to change it to `pl`. Training ++++++++ From 599e4580ef71b810bff9caa9a2198ca945616d65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire?= Date: Mon, 10 Nov 2014 18:42:37 -0500 Subject: [PATCH 102/417] add snippets --- code/rnnslu.py | 18 ++++++++-- doc/rnnslu.txt | 95 ++++++++++++-------------------------------------- 2 files changed, 38 insertions(+), 75 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index b8f2ceb2..12721b4a 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -30,11 +30,14 @@ def shuffle(lol, seed): random.seed(seed) random.shuffle(l) - +# start-snippet-1 def contextwin(l, win): ''' win :: int corresponding to the size of the window given a list of indexes composing a sentence + + l :: array containing the word indexes + it will return a list of list of indexes corresponding to context windows surrounding each word in the sentence ''' @@ -47,7 +50,7 @@ def contextwin(l, win): assert len(out) == len(l) return out - +# end-snippet-1 # data loading functions def atisfold(fold): @@ -122,7 +125,7 @@ def get_perf(filename): return {'p': precision, 'r': recall, 'f1': f1score} - +# start-snippet-2 class RNNSLU(object): ''' elman neural net model ''' def __init__(self, nh, nc, ne, de, cs): @@ -164,11 +167,14 @@ def __init__(self, nh, nc, ne, de, cs): # bundle self.params = [self.emb, self.wx, self.wh, self.w, self.bh, self.b, self.h0] + # end-snippet-2 # as many columns as context window size # as many lines as words in the sentence + # start-snippet-3 idxs = T.imatrix() x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) y_sentence = T.ivector('y_sentence') # labels + # end-snippet-3 start-snippet-4 def recurrence(x_t, h_tm1): h_t = T.nnet.sigmoid(T.dot(x_t, self.wx) @@ -183,28 +189,34 @@ def recurrence(x_t, h_tm1): p_y_given_x_sentence = s[:, 0, :] y_pred = T.argmax(p_y_given_x_sentence, axis=1) + # end-snippet-4 # cost and gradients and learning rate + # start-snippet-5 lr = T.scalar('lr') sentence_nll = -T.mean(T.log(p_y_given_x_sentence) [T.arange(x.shape[0]), y_sentence]) sentence_gradients = T.grad(sentence_nll, self.params) sentence_updates = OrderedDict((p, p - lr*g) + # end-snippet-5 for p, g in zip(self.params, sentence_gradients)) # theano functions to compile + # start-snippet-6 self.classify = theano.function(inputs=[idxs], outputs=y_pred) self.sentence_train = theano.function(inputs=[idxs, y_sentence, lr], outputs=sentence_nll, updates=sentence_updates) + # end-snippet-6 start-snippet-7 self.normalize = theano.function(inputs=[], updates={self.emb: self.emb / T.sqrt((self.emb**2) .sum(axis=1)) .dimshuffle(0, 'x')}) + # end-snippet-7 def train(self, x, y, window_size, learning_rate): diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 68f9b3a3..026dbc8b 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -128,26 +128,9 @@ Given a sentence i.e. an array of indexes, and a window size i.e. 1,3,5,..., we need to convert each word in the sentence to a context window surrounding this particular word. In details, we have:: - def contextwin(l, win): - ''' - win :: int corresponding to the size of the window - given a list of indexes composing a sentence - - l :: array containing the word indexes - - it will return a list of list of indexes corresponding - to context windows surrounding each word in the sentence - ''' - - assert (win % 2) == 1 - assert win >=1 - l = list(l) - - lpadded = win//2 * [-1] + l + win//2 * [-1] - out = [ lpadded[i:i+win] for i in range(len(l)) ] - - assert len(out) == len(l) - return out +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-1 + :end-before: end-snippet-1 The index ``-1`` corresponds to the ``PADDING`` index we insert at the beginning/end of the sentence. @@ -254,71 +237,39 @@ The **hyperparameters** define the whole architecture: It gives the following code:: - class RNNSLU(object): - - def __init__(self, nh, nc, ne, de, cs): - ''' - nh :: dimension of the hidden layer - nc :: number of classes - ne :: number of word embeddings in the vocabulary - de :: dimension of the word embeddings - cs :: word window context size - ''' - self.emb = theano.shared(name='embeddings', value=0.2 * numpy.random.uniform(-1.0, 1.0, - (ne+1, de)).astype(theano.config.floatX)) # add one for PADDING at the end - self.Wx = theano.shared(name='Wx', value=0.2 * numpy.random.uniform(-1.0, 1.0, - (de * cs, nh)).astype(theano.config.floatX)) - self.Wh = theano.shared(name='Wh', value=0.2 * numpy.random.uniform(-1.0, 1.0, - (nh, nh)).astype(theano.config.floatX)) - self.W = theano.shared(name='W', value=0.2 * numpy.random.uniform(-1.0, 1.0, - (nh, nc)).astype(theano.config.floatX)) - self.bh = theano.shared(name='bh', value=numpy.zeros(nh, dtype=theano.config.floatX)) - self.b = theano.shared(name='b', value=numpy.zeros(nc, dtype=theano.config.floatX)) - self.h0 = theano.shared(name='h0', value=numpy.zeros(nh, dtype=theano.config.floatX)) - - # bundle - self.params = [self.emb, self.Wx, self.Wh, self.W, self.bh, self.b, self.h0] +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-2 + :end-before: end-snippet-2 Then we integrate the way to build the input from the embedding matrix:: - idxs = T.imatrix() # as many columns as context window size/lines as words in the sentence - x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) - y = T.ivector('y') # label +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-3 + :end-before: end-snippet-3 We use the scan operator to construct the recursion, works like a charm:: - def recurrence(x_t, h_tm1): - h_t = T.nnet.sigmoid(T.dot(x_t, self.Wx) + T.dot(h_tm1, self.Wh) + self.bh) - s_t = T.nnet.softmax(T.dot(h_t, self.W) + self.b) - return [h_t, s_t] - - [h, s], _ = theano.scan(fn=recurrence, - sequences=x, outputs_info=[self.h0, None], - n_steps=x.shape[0]) - - p_y_given_x_sentence = s[:, 0, :] - y_pred = T.argmax(p_y_given_x_sentence, axis=1) +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-4 + :end-before: end-snippet-4 Theano will then compute all the gradients automatically to maximize the log-likelihood:: - lr = T.scalar('lr') - nll = -T.mean(T.log(p_y_given_x_sentence)[T.arange(x.shape[0]),y]) - gradients = T.grad( nll, self.params ) - updates = OrderedDict((p, p - lr*g) for p, g in zip(self.params, gradients)) - -Next compile those functions:: +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-5 + :end-before: end-snippet-5 - self.classify = theano.function(inputs=[idxs], outputs=y_pred) +Next compile those functions:: - self.train = theano.function(inputs=[idxs, y, lr], - outputs=nll, - updates=updates) +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-6 + :end-before: end-snippet-6 We keep the word embeddings on the unit sphere by normalizing them after each update:: - self.normalize = theano.function(inputs=[], - updates = {self.emb: - self.emb / T.sqrt((self.emb**2).sum(axis=1)).dimshuffle(0, 'x')}) +.. literalinclude:: ../code/rnnslu.py + :start-after: start-snippet-7 + :end-before: end-snippet-7 And that's it! @@ -373,7 +324,7 @@ The following intervals can give you some starting point: Running the Code ++++++++++++++++ -The user can then run the code by calling: +After downloading the data using `download.sh`, the user can then run the code by calling: .. code-block:: bash From 35ce5378f8081f9f1f42e67d6eb618c762179a69 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Tue, 11 Nov 2014 09:20:29 -0500 Subject: [PATCH 103/417] Do not consider "literalinclude" a code block --- doc/rnnslu.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 026dbc8b..503d6811 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -126,7 +126,7 @@ Context window Given a sentence i.e. an array of indexes, and a window size i.e. 1,3,5,..., we need to convert each word in the sentence to a context window surrounding this -particular word. In details, we have:: +particular word. In details, we have: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-1 @@ -235,37 +235,37 @@ The **hyperparameters** define the whole architecture: * number of classes * random seed + way to initialize the model -It gives the following code:: +It gives the following code: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-2 :end-before: end-snippet-2 -Then we integrate the way to build the input from the embedding matrix:: +Then we integrate the way to build the input from the embedding matrix: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-3 :end-before: end-snippet-3 -We use the scan operator to construct the recursion, works like a charm:: +We use the scan operator to construct the recursion, works like a charm: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-4 :end-before: end-snippet-4 -Theano will then compute all the gradients automatically to maximize the log-likelihood:: +Theano will then compute all the gradients automatically to maximize the log-likelihood: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-5 :end-before: end-snippet-5 -Next compile those functions:: +Next compile those functions: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-6 :end-before: end-snippet-6 -We keep the word embeddings on the unit sphere by normalizing them after each update:: +We keep the word embeddings on the unit sphere by normalizing them after each update: .. literalinclude:: ../code/rnnslu.py :start-after: start-snippet-7 From 8e7963e778a1b3a1b734d7d44972e488357335e6 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 12 Nov 2014 14:06:00 -0500 Subject: [PATCH 104/417] Pep8 / pyflakes --- code/rnnslu.py | 55 +++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 12721b4a..f6ddd094 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -30,6 +30,7 @@ def shuffle(lol, seed): random.seed(seed) random.shuffle(l) + # start-snippet-1 def contextwin(l, win): ''' @@ -45,13 +46,14 @@ def contextwin(l, win): assert win >= 1 l = list(l) - lpadded = win//2 * [-1] + l + win//2 * [-1] - out = [lpadded[i:i+win] for i in range(len(l))] + lpadded = win // 2 * [-1] + l + win // 2 * [-1] + out = [lpadded[i:(i + win)] for i in range(len(l))] assert len(out) == len(l) return out # end-snippet-1 + # data loading functions def atisfold(fold): assert fold in range(5) @@ -125,6 +127,7 @@ def get_perf(filename): return {'p': precision, 'r': recall, 'f1': f1score} + # start-snippet-2 class RNNSLU(object): ''' elman neural net model ''' @@ -199,9 +202,9 @@ def recurrence(x_t, h_tm1): [T.arange(x.shape[0]), y_sentence]) sentence_gradients = T.grad(sentence_nll, self.params) sentence_updates = OrderedDict((p, p - lr*g) - # end-snippet-5 for p, g in zip(self.params, sentence_gradients)) + # end-snippet-5 # theano functions to compile # start-snippet-6 @@ -238,28 +241,28 @@ def load(self, folder): param.name + '.npy'))) - def main(param=None): if not param: - param = {'fold': 3, - # 5 folds 0,1,2,3,4 - 'data': 'atis', - 'lr': 0.0970806646812754, - 'verbose': 1, - 'decay': True, - # decay on the learning rate if improvement stops - 'win': 7, - # number of words in the context window - 'nhidden': 200, - # number of hidden units - 'seed': 345, - 'emb_dimension': 50, - # dimension of word embedding - 'nepochs': 60, - # 60 is recommended - 'savemodel': False} + param = { + 'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 60, + # 60 is recommended + 'savemodel': False} print param - + folder = os.path.basename(__file__).split('.')[0] if not os.path.exists(folder): os.mkdir(folder) @@ -308,9 +311,11 @@ def main(param=None): for i, (x, y) in enumerate(zip(train_lex, train_y)): rnn.train(x, y, param['win'], param['clr']) - print '[learning] epoch %i >> %2.2f%%'%(e,(i+1)*100./nsentences),'completed in %.2f (sec) <<\r'%(time.time()-tic), + print '[learning] epoch %i >> %2.2f%%' % ( + e, (i + 1) * 100. / nsentences), + print 'completed in %.2f (sec) <<\r' % (time.time() - tic), sys.stdout.flush() - + # evaluation // back into the real world : idx -> words predictions_test = [map(lambda x: idx2label[x], rnn.classify(numpy.asarray( @@ -330,7 +335,7 @@ def main(param=None): groundtruth_valid, words_valid, folder + '/current.valid.txt') - + if res_valid['f1'] > best_f1: if param['savemodel']: From b64b5ad0b8e2ecced39c739d65a3de252ee15086 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 12 Nov 2014 17:49:55 -0500 Subject: [PATCH 105/417] Make path to data relative to script, not cwd --- code/rnnslu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index f6ddd094..0fe82230 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -15,7 +15,10 @@ import theano from theano import tensor as T -PREFIX = os.getenv('ATISDATA', 'data') +PREFIX = os.getenv( + 'ATISDATA', + os.path.join(os.path.split(os.path.abspath(os.path.dirname(__file__)))[0], + 'data')) # utils functions From bf5869291e3e5fd6420bf27c519a876b0bcbdbc2 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 12 Nov 2014 17:50:17 -0500 Subject: [PATCH 106/417] Always put conlleval.pl in code/rnnslu --- code/rnnslu.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 0fe82230..36ff13f1 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -67,7 +67,7 @@ def atisfold(fold): # metrics function using conlleval.pl -def conlleval(p, g, w, filename): +def conlleval(p, g, w, filename, script_path): ''' INPUT: p :: predictions @@ -79,6 +79,10 @@ def conlleval(p, g, w, filename): are written. it will be the input of conlleval.pl script for computing the performance in terms of precision recall and f1 score + + OTHER: + script_path :: path to the directory containing the + conlleval.pl script ''' out = '' for sl, sp, sw in zip(g, p, w): @@ -91,27 +95,26 @@ def conlleval(p, g, w, filename): f.writelines(out) f.close() - return get_perf(filename) + return get_perf(filename, script_path) -def download(origin): +def download(origin, destination): ''' download the corresponding atis file from http://www-etud.iro.umontreal.ca/~mesnilgr/atis/ ''' print 'Downloading data from %s' % origin - name = origin.split('/')[-1] - urllib.urlretrieve(origin, name) + urllib.urlretrieve(origin, destination) -def get_perf(filename): +def get_perf(filename, folder): ''' run conlleval.pl perl script to obtain precision/recall and F1 score ''' - _conlleval = 'conlleval.pl' + _conlleval = os.path.join(folder, 'conlleval.pl') if not os.path.isfile(_conlleval): url = 'http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl' - download(url) - os.chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions + download(url, _conlleval) + os.chmod(_conlleval, stat.S_IRWXU) # give the execute permissions proc = subprocess.Popen(["perl", _conlleval], @@ -333,11 +336,13 @@ def main(param=None): res_test = conlleval(predictions_test, groundtruth_test, words_test, - folder + '/current.test.txt') + folder + '/current.test.txt', + folder) res_valid = conlleval(predictions_valid, groundtruth_valid, words_valid, - folder + '/current.valid.txt') + folder + '/current.valid.txt', + folder) if res_valid['f1'] > best_f1: From 9b4f9d38724789843dd6eba6866a6904c42d1022 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 12 Nov 2014 17:57:57 -0500 Subject: [PATCH 107/417] Always create rnnslu/ inside code/, not in cwd --- code/rnnslu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 36ff13f1..4ab80b54 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -269,7 +269,8 @@ def main(param=None): 'savemodel': False} print param - folder = os.path.basename(__file__).split('.')[0] + folder_name = os.path.basename(__file__).split('.')[0] + folder = os.path.join(os.path.dirname(__file__), folder_name) if not os.path.exists(folder): os.mkdir(folder) From 52bcea645482ae535f6ff5350e1db8da8b3f72ba Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 12 Nov 2014 17:58:20 -0500 Subject: [PATCH 108/417] Ignore automatically-generated or downloaded files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3392e83c..f891e40e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ code/*.pyc +code/*_plots code/tmp* code/midi +code/rnnslu +data/atis.* data/mnist.pkl.gz data/mnist_py3k.pkl.gz data/Nottingham.zip From c1f65d2605f23b1b1d3309a6280812867eb4c809 Mon Sep 17 00:00:00 2001 From: Ivan Ukhov Date: Sun, 14 Dec 2014 17:59:31 +0100 Subject: [PATCH 109/417] Fix a typo in LeNet Sparse Connectivity --- doc/lenet.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 56636d9f..8047cb28 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -86,13 +86,13 @@ graphically as follows: Imagine that layer **m-1** is the input retina. In the above figure, units in layer **m** have receptive fields of width 3 in the input retina and are thus -only connected to 3 adjacent neurons in the retina layer. Units in layer **m** -have a similar connectivity with the layer below. We say that their receptive -field with respect to the layer below is also 3, but their receptive field with -respect to the input is larger (5). Each unit is unresponsive to variations -outside of its receptive field with respect to the retina. The architecture -thus ensures that the learnt "filters" produce the strongest response to a -spatially local input pattern. +only connected to 3 adjacent neurons in the retina layer. Units in layer +**m+1** have a similar connectivity with the layer below. We say that their +receptive field with respect to the layer below is also 3, but their receptive +field with respect to the input is larger (5). Each unit is unresponsive to +variations outside of its receptive field with respect to the retina. The +architecture thus ensures that the learnt "filters" produce the strongest +response to a spatially local input pattern. However, as shown above, stacking many such layers leads to (non-linear) "filters" that become increasingly "global" (i.e. responsive to a larger region From d788ebc868dc3a706521175ec2ccb21180db3842 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Fri, 19 Dec 2014 11:21:19 -0500 Subject: [PATCH 110/417] Add basic LSTM tutorial --- code/imdb.py | 92 ++++++++ code/lstm.py | 578 +++++++++++++++++++++++++++++++++++++++++++++++ data/download.sh | 1 + doc/intro.txt | 3 + 4 files changed, 674 insertions(+) create mode 100644 code/imdb.py create mode 100644 code/lstm.py diff --git a/code/imdb.py b/code/imdb.py new file mode 100644 index 00000000..830dc872 --- /dev/null +++ b/code/imdb.py @@ -0,0 +1,92 @@ +import cPickle +import gzip +import os +import sys +import time + +import numpy + +import theano +import theano.tensor as T + +def prepare_data(seqs, labels, maxlen=None): + # x: a list of sentences + lengths = [len(s) for s in seqs] + + if maxlen != None: + new_seqs = [] + new_labels = [] + new_lengths = [] + for l, s, y in zip(lengths, seqs, labels): + if l < maxlen: + new_seqs.append(s) + new_labels.append(y) + new_lengths.append(l) + lengths = new_lengths + labels = new_labels + seqs = new_seqs + + if len(lengths) < 1: + return None, None, None + + n_samples = len(seqs) + maxlen = numpy.max(lengths) + + x = numpy.zeros((maxlen, n_samples)).astype('int64') + x_mask = numpy.zeros((maxlen, n_samples)).astype('float32') + for idx, s in enumerate(seqs): + x[:lengths[idx],idx] = s + x_mask[:lengths[idx],idx] = 1. + + return x, x_mask, labels + +def load_data(path="/data/lisatmp3/chokyun/tweets_sa/imdb/aclImdb/imdb.pkl", n_words=100000, valid_portion=0.1): + ''' Loads the dataset + + :type dataset: string + :param dataset: the path to the dataset (here IMDB) + ''' + + ############# + # LOAD DATA # + ############# + + print '... loading data' + + # Load the dataset + f = open(path, 'rb') + train_set = cPickle.load(f) + test_set = cPickle.load(f) + f.close() + + # split training set into validation set + train_set_x, train_set_y = train_set + n_samples = len(train_set_x) + sidx = numpy.random.permutation(n_samples) + n_train = int(numpy.round(n_samples * (1. - valid_portion))) + valid_set_x = [train_set_x[s] for s in sidx[n_train:]] + valid_set_y = [train_set_y[s] for s in sidx[n_train:]] + train_set_x = [train_set_x[s] for s in sidx[:n_train]] + train_set_y = [train_set_y[s] for s in sidx[:n_train]] + + train_set = (train_set_x, train_set_y) + valid_set = (valid_set_x, valid_set_y) + + def remove_unk(x): + return [[1 if w >= n_words else w for w in sen] for sen in x] + + test_set_x, test_set_y = test_set + valid_set_x, valid_set_y = valid_set + train_set_x, train_set_y = train_set + + train_set_x = remove_unk(train_set_x) + valid_set_x = remove_unk(valid_set_x) + test_set_x = remove_unk(test_set_x) + + train = (train_set_x, train_set_y) + valid = (valid_set_x, valid_set_y) + test = (test_set_x, test_set_y) + + return train, valid, test + + diff --git a/code/lstm.py b/code/lstm.py new file mode 100644 index 00000000..74f6ab55 --- /dev/null +++ b/code/lstm.py @@ -0,0 +1,578 @@ +''' +Build a tweet sentiment analyzer +''' +import theano +import theano.tensor as tensor +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams + +import cPickle as pkl +import numpy +import copy +import random + +from collections import OrderedDict + +import imdb + +datasets = {'imdb': (imdb.load_data, imdb.prepare_data)} + +def get_minibatches_idx(n, nb_batches, shuffle=False): + + idx_list = numpy.arange(n, dtype="int32") + + if shuffle: + random.shuffle(idx_list) + + minibatches = [] + minibatch_start = 0 + for i in range(nb_batches): + if i < n % nb_batches: + minibatch_size = n // nb_batches + 1 + else: + minibatch_size = n // nb_batches + + + minibatches.append(idx_list[minibatch_start: + minibatch_start + minibatch_size]) + minibatch_start += minibatch_size + + return zip(range(nb_batches), minibatches) + +def get_dataset(name): + return datasets[name][0], datasets[name][1] + +def zipp(params, tparams): + for kk, vv in params.iteritems(): + tparams[kk].set_value(vv) + +def unzip(zipped): + new_params = OrderedDict() + for kk, vv in zipped.iteritems(): + new_params[kk] = vv.get_value() + return new_params + +def itemlist(tparams): + return [vv for kk, vv in tparams.iteritems()] + +def dropout_layer(state_before, use_noise, trng): + proj = tensor.switch(use_noise, + state_before * trng.binomial(state_before.shape, p=0.5, n=1, dtype=state_before.dtype), + state_before * 0.5) + return proj + +def _p(pp, name): + return '%s_%s'%(pp, name) + +def init_params(options): + params = OrderedDict() + # embedding + params['Wemb'] = 0.01 * numpy.random.randn(options['n_words'], options['dim_proj']).astype('float32') + # rconv + params = get_layer(options['encoder'])[0](options, params, prefix=options['encoder']) + # classifier + params['U'] = 0.01 * numpy.random.randn(options['dim_proj'], options['ydim']).astype('float32') + params['b'] = numpy.zeros((options['ydim'],)).astype('float32') + + return params + +def load_params(path, params): + pp = numpy.load(path) + for kk, vv in params.iteritems(): + if kk not in pp: + raise Warning('%s is not in the archive'%kk) + params[kk] = pp[kk] + + return params + +def init_tparams(params): + tparams = OrderedDict() + for kk, pp in params.iteritems(): + tparams[kk] = theano.shared(params[kk], name=kk) + return tparams + +layers = {'ff': ('param_init_fflayer', 'fflayer'), + 'rconv': ('param_init_rconv', 'rconv_layer'), + 'lstm': ('param_init_lstm', 'lstm_layer')} + +def get_layer(name): + fns = layers[name] + return (eval(fns[0]), eval(fns[1])) + + +def param_init_fflayer(options, params, prefix='ff'): + params[_p(prefix,'W')] = 0.01 * numpy.random.randn(options['dim_proj'], options['dim_proj']).astype('float32') + params[_p(prefix,'b')] = numpy.zeros((options['dim_proj'],)).astype('float32') + + return params + +def fflayer(tparams, state_below, options, prefix='rconv', **kwargs): + return eval(options['activ'])(tensor.dot(state_below, tparams[_p(prefix,'W')])+tparams[_p(prefix,'b')]) + +def ortho_weight(ndim): + W = numpy.random.randn(ndim, ndim) + u, s, v = numpy.linalg.svd(W) + return u.astype('float32') + +def param_init_lstm(options, params, prefix='lstm'): + W = numpy.concatenate([ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj'])], axis=1) + params[_p(prefix,'W')] = W + U = numpy.concatenate([ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj']), + ortho_weight(options['dim_proj'])], axis=1) + params[_p(prefix,'U')] = U + params[_p(prefix,'b')] = numpy.zeros((4 * options['dim_proj'],)).astype('float32') + + return params + +def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None): + nsteps = state_below.shape[0] + if state_below.ndim == 3: + n_samples = state_below.shape[1] + else: + n_samples = 1 + + if mask == None: + mask = tensor.alloc(1., x.shape[0], 1) + + def _slice(_x, n, dim): + if _x.ndim == 3: + return _x[:, :, n*dim:(n+1)*dim] + return _x[:, n*dim:(n+1)*dim] + + def _step(m_, x_, h_, c_): + preact = tensor.dot(h_, tparams[_p(prefix, 'U')]) + preact += x_ + preact += tparams[_p(prefix, 'b')] + + i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj'])) + f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj'])) + o = tensor.nnet.sigmoid(_slice(preact, 2, options['dim_proj'])) + c = tensor.tanh(_slice(preact, 3, options['dim_proj'])) + + c = f * c_ + i * c + c = m_[:,None] * c + (1. - m_)[:,None] * c_ + + h = o * tensor.tanh(c) + h = m_[:,None] * h + (1. - m_)[:,None] * h_ + + return h, c + + state_below = tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')] + + rval, updates = theano.scan(_step, + sequences=[mask, state_below], + outputs_info = [tensor.alloc(0., n_samples, options['dim_proj']), + tensor.alloc(0., n_samples, options['dim_proj'])], + name=_p(prefix, '_layers'), + n_steps=nsteps) + return rval[0] + +def param_init_rconv(options, params, prefix='rconv'): + params[_p(prefix,'W')] = ortho_weight(options['dim_proj']) + params[_p(prefix,'U')] = ortho_weight(options['dim_proj']) + params[_p(prefix,'b')] = numpy.zeros((options['dim_proj'],)).astype('float32') + params[_p(prefix,'GW')] = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') + params[_p(prefix,'GU')] = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') + params[_p(prefix,'Gb')] = numpy.zeros((3,)).astype('float32') + + return params + +def rconv_layer(tparams, state_below, options, prefix='rconv', mask=None): + nsteps = state_below.shape[0] + + if mask == None: + mask = tensor.alloc(1., x.shape[0], 1) + + def _step(m_, p_): + l_ = p_ + # new activation + ps_ = tensor.zeros_like(p_) + ps_ = tensor.set_subtensor(ps_[1:], p_[:-1]) + ls_ = ps_ + ps_ = tensor.dot(ps_, tparams[_p(prefix, 'U')]) + pl_ = tensor.dot(p_, tparams[_p(prefix, 'W')]) + newact = eval(options['activ'])(ps_+pl_+tparams[_p(prefix, 'b')]) + + # gater + gt_ = tensor.dot(ls_, tparams[_p(prefix, 'GU')]) +\ + tensor.dot(l_, tparams[_p(prefix, 'GW')]) + \ + tparams[_p(prefix, 'Gb')] + if l_.ndim == 3: + gt_shp = gt_.shape + gt_ = gt_.reshape((gt_shp[0]*gt_shp[1],gt_shp[2])) + gt_ = tensor.nnet.softmax(gt_) + if l_.ndim == 3: + gt_ = gt_.reshape((gt_shp[0],gt_shp[1],gt_shp[2])) + + if p_.ndim == 3: + gn = gt_[:,:,0].dimshuffle(0,1,'x') + gl = gt_[:,:,1].dimshuffle(0,1,'x') + gr = gt_[:,:,2].dimshuffle(0,1,'x') + else: + gn = gt_[:,0].dimshuffle(0,'x') + gl = gt_[:,1].dimshuffle(0,'x') + gr = gt_[:,2].dimshuffle(0,'x') + + act = newact * gn + ls_ * gl + l_ * gr + + if p_.ndim == 3: + m_ = m_.dimshuffle('x',0,'x') + else: + m_ = m_.dimshuffle('x', 0) + return tensor.switch(m_, act, l_) + + rval, updates = theano.scan(_step, + sequences = [mask[1:]], + outputs_info = [state_below], + name='layer_%s'%prefix, + n_steps = nsteps-1) + + seqlens = tensor.cast(mask.sum(axis=0), 'int64')-1 + roots = rval[-1] + + if state_below.ndim == 3: + def _grab_root(seqlen,one_sample,prev_sample): + return one_sample[seqlen] + + roots, updates = theano.scan(_grab_root, + sequences = [seqlens, roots.dimshuffle(1,0,2)], + outputs_info = [tensor.alloc(0., options['dim_proj'])], + name='grab_root_%s'%prefix) + #roots = roots.dimshuffle('x', 0, 1) + else: + roots = roots[seqlens] # there should be only one, so it's fine. + + return roots + +def adadelta(lr, tparams, grads, x, mask, y, cost): + zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()] + running_up2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rup2'%k) for k, p in tparams.iteritems()] + running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2'%k) for k, p in tparams.iteritems()] + + zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] + rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] + + f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rg2up) + + updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, running_up2, running_grads2)] + ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] + param_up = [(p, p + ud) for p, ud in zip(itemlist(tparams), updir)] + + f_update = theano.function([lr], [], updates=ru2up+param_up, on_unused_input='ignore') + + return f_grad_shared, f_update + +def rmsprop(lr, tparams, grads, x, mask, y, cost): + zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()] + running_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad'%k) for k, p in tparams.iteritems()] + running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2'%k) for k, p in tparams.iteritems()] + + zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] + rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] + rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] + + f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rgup+rg2up) + + updir = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_updir'%k) for k, p in tparams.iteritems()] + updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] + param_up = [(p, p + udn[1]) for p, udn in zip(itemlist(tparams), updir_new)] + f_update = theano.function([lr], [], updates=updir_new+param_up, on_unused_input='ignore') + + return f_grad_shared, f_update + +def sgd(lr, tparams, grads, x, mask, y, cost): + gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) for k, p in tparams.iteritems()] + gsup = [(gs, g) for gs, g in zip(gshared, grads)] + + f_grad_shared = theano.function([x, mask, y], cost, updates=gsup) + + pup = [(p, p - lr * g) for p, g in zip(itemlist(tparams), gshared)] + f_update = theano.function([lr], [], updates=pup) + + return f_grad_shared, f_update + +def build_model(tparams, options): + trng = RandomStreams(1234) + use_noise = theano.shared(numpy.float32(0.)) + + x = tensor.matrix('x', dtype='int64') + mask = tensor.matrix('mask', dtype='float32') + y = tensor.vector('y', dtype='int64') + + n_timesteps = x.shape[0] + n_samples = x.shape[1] + + emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, n_samples, options['dim_proj']]) + proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix=options['encoder'], mask=mask) + if options['encoder'] == 'lstm': + #proj = proj[-1] + #proj = proj.mean(axis=0) + proj = (proj * mask[:,:,None]).sum(axis=0) + proj = proj / mask.sum(axis=0)[:,None] + if options['use_dropout']: + proj = dropout_layer(proj, use_noise, trng) + + pred = tensor.nnet.softmax(tensor.dot(proj, tparams['U'])+tparams['b']) + + f_pred_prob = theano.function([x, mask], pred) + f_pred = theano.function([x, mask], pred.argmax(axis=1)) + + cost = -tensor.log(pred[tensor.arange(n_samples), y]+1e-8).mean() + + return trng, use_noise, x, mask, y, f_pred_prob, f_pred, cost + +def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): + n_samples = len(data[0]) + probs = numpy.zeros((n_samples, 2)).astype('float32') + + n_done = 0 + + for _, valid_index in iterator: + x, mask, y = prepare_data([data[0][t] for t in valid_index], numpy.array(data[1])[valid_index], maxlen=None) + pred_probs = f_pred_prob(x,mask) + probs[valid_index,:] = pred_probs + + n_done += len(valid_index) + if verbose: + print '%d/%d samples classified'%(n_done,n_samples) + + return probs + +def pred_error(f_pred, prepare_data, data, iterator, verbose=False): + valid_err = 0 + #valid_class_counts = numpy.zeros(3) + #valid_class_corrects = numpy.zeros(3) + for _, valid_index in iterator: + x, mask, y = prepare_data([data[0][t] for t in valid_index], numpy.array(data[1])[valid_index], maxlen=None) + preds = f_pred(x,mask) + targets = numpy.array(data[1])[valid_index] + #for cc in xrange(3): + # valid_class_counts[cc] += numpy.sum(targets == cc) + # valid_class_corrects[cc] += (preds[numpy.where(targets == cc)] == targets[numpy.where(targets == cc)]).sum() + valid_err += (preds == targets).sum() + #valid_err = 1. - (valid_class_corrects.astype('float32') / valid_class_counts.astype('float32')).mean() + valid_err = 1. - numpy.float32(valid_err) / len(data[0]) + + return valid_err + + +def train(dim_proj=100, + patience=10, + max_epochs = 5000, + dispFreq=100, + activ='lambda x: tensor.tanh(x)', + decay_c = 0., + lrate=0.01, + n_words=100000, + data_sym=False, + optimizer='rmsprop', + encoder='rconv', + saveto='model.npz', + noise_std=0., + validFreq=1000, + saveFreq=1000, # save the parameters after every saveFreq updates + maxlen=50, + batch_size = 16, + valid_batch_size = 16, + dataset='sentiment140', + use_dropout=False): + + # Model options + model_options = locals().copy() + + load_data, prepare_data = get_dataset(dataset) + + print 'Loading data' + train, valid, test = load_data(n_words=n_words, valid_portion=0.01) + + ydim = numpy.max(train[1])+1 + + model_options['ydim'] = ydim + + print 'Building model' + params = init_params(model_options) + tparams = init_tparams(params) + + trng, use_noise, \ + x, mask, y, \ + f_pred_prob, f_pred, cost = \ + build_model(tparams, model_options) + + if decay_c > 0.: + decay_c = theano.shared(numpy.float32(decay_c), name='decay_c') + weight_decay = 0. + weight_decay += (tparams['U']**2).sum() + weight_decay *= decay_c + cost += weight_decay + + f_cost = theano.function([x, mask, y], cost) + + grads = tensor.grad(cost, wrt=itemlist(tparams)) + f_grad = theano.function([x, mask, y], grads) + + lr = tensor.scalar(name='lr') + f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, x, mask, y, cost) + + print 'Optimization' + + kf_valid = get_minibatches_idx(len(valid[0]), len(valid[0])/valid_batch_size, shuffle=True) + kf_test = get_minibatches_idx(len(test[0]), len(test[0])/valid_batch_size, shuffle=True) + + + history_errs = [] + best_p = None + bad_count = 0 + + if validFreq == -1: + validFreq = len(train[0])/batch_size + if saveFreq == -1: + saveFreq = len(train[0])/batch_size + + uidx = 0 + estop = False + for eidx in xrange(max_epochs): + n_samples = 0 + + kf = get_minibatches_idx(len(train[0]), len(train[0])/batch_size, shuffle=True) + + for _, train_index in kf: + n_samples += train_index.shape[0] + uidx += 1 + use_noise.set_value(1.) + + y = [train[1][t] for t in train_index] + x, mask, y = prepare_data([train[0][t] for t in train_index], y, maxlen=maxlen) + + if x == None: + print 'Minibatch with zero sample under length ', maxlen + continue + + cost = f_grad_shared(x, mask, y) + f_update(lrate) + + if numpy.isnan(cost) or numpy.isinf(cost): + print 'NaN detected' + return 1., 1., 1. + + if numpy.mod(uidx, dispFreq) == 0: + print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost + + if numpy.mod(uidx, saveFreq) == 0: + print 'Saving...', + + #import ipdb; ipdb.set_trace() + + if best_p != None: + params = best_p + else: + params = unzip(tparams) + numpy.savez(saveto, history_errs=history_errs, **params) + pkl.dump(model_options, open('%s.pkl'%saveto, 'wb')) + print 'Done' + + if numpy.mod(uidx, validFreq) == 0: + use_noise.set_value(0.) + train_err = 0 + #for _, tindex in kf: + # x, mask = prepare_data(train[0][train_index]) + # train_err += (f_pred(x, mask) == train[1][tindex]).sum() + #train_err = 1. - numpy.float32(train_err) / train[0].shape[0] + + #train_err = pred_error(f_pred, prepare_data, train, kf) + valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) + test_err = pred_error(f_pred, prepare_data, test, kf_test) + + history_errs.append([valid_err, test_err]) + + if uidx == 0 or valid_err <= numpy.array(history_errs)[:,0].min(): + best_p = unzip(tparams) + bad_counter = 0 + if eidx > patience and valid_err >= numpy.array(history_errs)[:-patience,0].min(): + bad_counter += 1 + if bad_counter > patience: + print 'Early Stop!' + estop = True + break + + print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err + + #print 'Epoch ', eidx, 'Update ', uidx, 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err + + print 'Seen %d samples'%n_samples + + if estop: + break + + + + if best_p is not None: + zipp(best_p, tparams) + + use_noise.set_value(0.) + train_err = 0 + #train_err = pred_error(f_pred, prepare_data, train, kf) + valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) + test_err = pred_error(f_pred, prepare_data, test, kf_test) + + + print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err + + params = copy.copy(best_p) + numpy.savez(saveto, zipped_params=best_p, train_err=train_err, + valid_err=valid_err, test_err=test_err, history_errs=history_errs, + **params) + + return train_err, valid_err, test_err + + +def main(job_id, params): + print 'Anything printed here will end up in the output directory for job #%d' % job_id + print params + trainerr, validerr, testerr = train(saveto=params['model'][0], + dim_proj=params['dim-proj'][0], + n_words=params['n-words'][0], + decay_c=params['decay-c'][0], + lrate=params['learning-rate'][0], + optimizer=params['optimizer'][0], + activ=params['activ'][0], + encoder=params['encoder'][0], + maxlen=600, + batch_size=16, + valid_batch_size=16, + validFreq=10000, + dispFreq=10, + saveFreq=100000, + dataset='imdb', #'stanford', #'sentiment140', + use_dropout=True if params['use-dropout'][0] else False) + return validerr + +if __name__ == '__main__': + main(0, { + 'model': ['model_lstm.npz'], + 'encoder': ['lstm'], + 'dim-proj': [128], + 'n-words': [10000], + 'optimizer': ['adadelta'], + #'activ': ['lambda x: tensor.maximum(0.,x)'], + 'activ': ['lambda x: tensor.tanh(x)'], + 'decay-c': [0.], + 'use-dropout': [1], + 'learning-rate': [0.0001]}) + + + + + + + + + + + + + + diff --git a/data/download.sh b/data/download.sh index e2a837b9..8a8e9a92 100755 --- a/data/download.sh +++ b/data/download.sh @@ -15,6 +15,7 @@ fi $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" $DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz diff --git a/doc/intro.txt b/doc/intro.txt index bd43434f..86390e20 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -52,6 +52,9 @@ Building towards including the Contractive auto-encoders tutorial, we have the c Recurrent neural networks with word embeddings and context window: * :ref:`Semantic Parsing of Speech using Recurrent Net ` +LSTM network for sentiment analysis: + * :ref:`LSTM network ` - Only the code for now + Energy-based recurrent neural network (RNN-RBM): * :ref:`Modeling and generating sequences of polyphonic music ` From 062484905d181c5e49a0ff8b86c1093a3d70be9b Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Fri, 19 Dec 2014 12:44:56 -0500 Subject: [PATCH 111/417] Add lstm.doc and update dataset path --- code/imdb.py | 2 +- doc/lstm.txt | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 doc/lstm.txt diff --git a/code/imdb.py b/code/imdb.py index 830dc872..f063ccea 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -40,7 +40,7 @@ def prepare_data(seqs, labels, maxlen=None): return x, x_mask, labels -def load_data(path="/data/lisatmp3/chokyun/tweets_sa/imdb/aclImdb/imdb.pkl", n_words=100000, valid_portion=0.1): +def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): ''' Loads the dataset :type dataset: string diff --git a/doc/lstm.txt b/doc/lstm.txt new file mode 100644 index 00000000..9bc53e62 --- /dev/null +++ b/doc/lstm.txt @@ -0,0 +1,65 @@ +.. _lstm: + +Recurrent Neural Networks with Word Embeddings +********************************************** + +Summary ++++++++ + +This tutorial aims to provide an example of how a Recurrent Neural Network (RNN) using the Long Short Term Memory (LSTM) architecture can be implemented using Theano. In this tutorial, this model is used to perform sentiment analysis on movie reviews from the `Large Movie Review Dataset `_, sometimes known as the IMDB dataset. + +In this task, given a movie review, the model attempts to predict whether it is positive or negative. This is a binary classification task. + +Code - Citations - Contact +++++++++++++++++++++++++++ + +Code +==== + +The LSTM implementation can be found in the two following files : + +* `lstm.py `_ : Main script. Defines and train the model. + +* `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. + +Data +==== + +As previously mentionned, the provided scripts are used to train a LSTM +recurrent neural on the Large Movie Review Dataset dataset. + +While the dataset is public, in this tutorial we provide a copy of the dataset +that has previously been preprocessed according to the needs of this LSTM +implementation. You can download this preprocessed version of the dataset +using the script `download.sh` and uncompress it. + +Papers +====== + +If you use this tutorial, please cite the following papers: + +* `[pdf] `_ HOCHREITER, Sepp et SCHMIDHUBER, Jürgen. Long short-term memory. Neural computation, 1997, vol. 9, no 8, p. 1735-1780. 1997. + +* `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. + +* `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. + +Thank you! + +Contact +======= + +Please email `Kyunghyun Cho `_ for any +problem report or feedback. We will be glad to hear from you. + +Running the Code +++++++++++++++++ + +After downloading both the scripts, downloading and uncompressing the data and +putting all those files in the same folder, the user can run the code by +calling: + +.. code-block:: bash + + THEANO_FLAGS="floatX=float32" python train_lstm.py + From 7d804627396946961c2c3d4fc6062a79770eed8f Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Fri, 19 Dec 2014 12:48:21 -0500 Subject: [PATCH 112/417] Removed comments --- code/lstm.py | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 74f6ab55..a0d1d30f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -242,7 +242,6 @@ def _grab_root(seqlen,one_sample,prev_sample): sequences = [seqlens, roots.dimshuffle(1,0,2)], outputs_info = [tensor.alloc(0., options['dim_proj'])], name='grab_root_%s'%prefix) - #roots = roots.dimshuffle('x', 0, 1) else: roots = roots[seqlens] # there should be only one, so it's fine. @@ -309,8 +308,6 @@ def build_model(tparams, options): emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, n_samples, options['dim_proj']]) proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix=options['encoder'], mask=mask) if options['encoder'] == 'lstm': - #proj = proj[-1] - #proj = proj.mean(axis=0) proj = (proj * mask[:,:,None]).sum(axis=0) proj = proj / mask.sum(axis=0)[:,None] if options['use_dropout']: @@ -344,17 +341,11 @@ def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): def pred_error(f_pred, prepare_data, data, iterator, verbose=False): valid_err = 0 - #valid_class_counts = numpy.zeros(3) - #valid_class_corrects = numpy.zeros(3) for _, valid_index in iterator: x, mask, y = prepare_data([data[0][t] for t in valid_index], numpy.array(data[1])[valid_index], maxlen=None) preds = f_pred(x,mask) targets = numpy.array(data[1])[valid_index] - #for cc in xrange(3): - # valid_class_counts[cc] += numpy.sum(targets == cc) - # valid_class_corrects[cc] += (preds[numpy.where(targets == cc)] == targets[numpy.where(targets == cc)]).sum() valid_err += (preds == targets).sum() - #valid_err = 1. - (valid_class_corrects.astype('float32') / valid_class_counts.astype('float32')).mean() valid_err = 1. - numpy.float32(valid_err) / len(data[0]) return valid_err @@ -464,8 +455,6 @@ def train(dim_proj=100, if numpy.mod(uidx, saveFreq) == 0: print 'Saving...', - #import ipdb; ipdb.set_trace() - if best_p != None: params = best_p else: @@ -476,13 +465,7 @@ def train(dim_proj=100, if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) - train_err = 0 - #for _, tindex in kf: - # x, mask = prepare_data(train[0][train_index]) - # train_err += (f_pred(x, mask) == train[1][tindex]).sum() - #train_err = 1. - numpy.float32(train_err) / train[0].shape[0] - - #train_err = pred_error(f_pred, prepare_data, train, kf) + train_err = pred_error(f_pred, prepare_data, train, kf) valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) @@ -500,8 +483,6 @@ def train(dim_proj=100, print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err - #print 'Epoch ', eidx, 'Update ', uidx, 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err - print 'Seen %d samples'%n_samples if estop: @@ -513,8 +494,7 @@ def train(dim_proj=100, zipp(best_p, tparams) use_noise.set_value(0.) - train_err = 0 - #train_err = pred_error(f_pred, prepare_data, train, kf) + train_err = pred_error(f_pred, prepare_data, train, kf) valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) @@ -557,7 +537,6 @@ def main(job_id, params): 'dim-proj': [128], 'n-words': [10000], 'optimizer': ['adadelta'], - #'activ': ['lambda x: tensor.maximum(0.,x)'], 'activ': ['lambda x: tensor.tanh(x)'], 'decay-c': [0.], 'use-dropout': [1], From 33914a695fec8bde2976f97e73e9d57fe1c1b89d Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Fri, 19 Dec 2014 15:57:23 -0500 Subject: [PATCH 113/417] pep8 --- code/lstm.py | 352 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 212 insertions(+), 140 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index a0d1d30f..a520f08f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -16,13 +16,14 @@ datasets = {'imdb': (imdb.load_data, imdb.prepare_data)} + def get_minibatches_idx(n, nb_batches, shuffle=False): idx_list = numpy.arange(n, dtype="int32") if shuffle: random.shuffle(idx_list) - + minibatches = [] minibatch_start = 0 for i in range(nb_batches): @@ -30,104 +31,130 @@ def get_minibatches_idx(n, nb_batches, shuffle=False): minibatch_size = n // nb_batches + 1 else: minibatch_size = n // nb_batches - - + minibatches.append(idx_list[minibatch_start: minibatch_start + minibatch_size]) minibatch_start += minibatch_size - + return zip(range(nb_batches), minibatches) + def get_dataset(name): return datasets[name][0], datasets[name][1] + def zipp(params, tparams): for kk, vv in params.iteritems(): tparams[kk].set_value(vv) + def unzip(zipped): new_params = OrderedDict() for kk, vv in zipped.iteritems(): new_params[kk] = vv.get_value() return new_params + def itemlist(tparams): return [vv for kk, vv in tparams.iteritems()] + def dropout_layer(state_before, use_noise, trng): - proj = tensor.switch(use_noise, - state_before * trng.binomial(state_before.shape, p=0.5, n=1, dtype=state_before.dtype), - state_before * 0.5) + proj = tensor.switch(use_noise, + (state_before * + trng.binomial(state_before.shape, + p=0.5, n=1, + dtype=state_before.dtype)), + state_before * 0.5) return proj + def _p(pp, name): - return '%s_%s'%(pp, name) + return '%s_%s' % (pp, name) + def init_params(options): params = OrderedDict() # embedding - params['Wemb'] = 0.01 * numpy.random.randn(options['n_words'], options['dim_proj']).astype('float32') + randn = numpy.random.rand(options['n_words'], + options['dim_proj']) + params['Wemb'] = (0.01 * randn).astype('float32') # rconv - params = get_layer(options['encoder'])[0](options, params, prefix=options['encoder']) + params = get_layer(options['encoder'])[0](options, + params, + prefix=options['encoder']) # classifier - params['U'] = 0.01 * numpy.random.randn(options['dim_proj'], options['ydim']).astype('float32') + params['U'] = 0.01 * numpy.random.randn(options['dim_proj'], + options['ydim']).astype('float32') params['b'] = numpy.zeros((options['ydim'],)).astype('float32') return params + def load_params(path, params): pp = numpy.load(path) for kk, vv in params.iteritems(): if kk not in pp: - raise Warning('%s is not in the archive'%kk) + raise Warning('%s is not in the archive' % kk) params[kk] = pp[kk] return params + def init_tparams(params): tparams = OrderedDict() for kk, pp in params.iteritems(): tparams[kk] = theano.shared(params[kk], name=kk) return tparams -layers = {'ff': ('param_init_fflayer', 'fflayer'), +layers = {'ff': ('param_init_fflayer', 'fflayer'), 'rconv': ('param_init_rconv', 'rconv_layer'), 'lstm': ('param_init_lstm', 'lstm_layer')} + def get_layer(name): fns = layers[name] return (eval(fns[0]), eval(fns[1])) def param_init_fflayer(options, params, prefix='ff'): - params[_p(prefix,'W')] = 0.01 * numpy.random.randn(options['dim_proj'], options['dim_proj']).astype('float32') - params[_p(prefix,'b')] = numpy.zeros((options['dim_proj'],)).astype('float32') + weights = numpy.random.randn(options['dim_proj'], options['dim_proj']) + biases = numpy.zeros((options['dim_proj'], )) + params[_p(prefix, 'W')] = 0.01 * weights.astype('float32') + params[_p(prefix, 'b')] = biases.astype('float32') return params + def fflayer(tparams, state_below, options, prefix='rconv', **kwargs): - return eval(options['activ'])(tensor.dot(state_below, tparams[_p(prefix,'W')])+tparams[_p(prefix,'b')]) + pre_act = (tensor.dot(state_below, + tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]) + return eval(options['activ'])(pre_act) + def ortho_weight(ndim): W = numpy.random.randn(ndim, ndim) u, s, v = numpy.linalg.svd(W) return u.astype('float32') + def param_init_lstm(options, params, prefix='lstm'): W = numpy.concatenate([ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj'])], axis=1) - params[_p(prefix,'W')] = W + params[_p(prefix, 'W')] = W U = numpy.concatenate([ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj'])], axis=1) - params[_p(prefix,'U')] = U - params[_p(prefix,'b')] = numpy.zeros((4 * options['dim_proj'],)).astype('float32') + params[_p(prefix, 'U')] = U + b = numpy.zeros((4 * options['dim_proj'],)) + params[_p(prefix, 'b')] = b.astype('float32') return params + def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None): nsteps = state_below.shape[0] if state_below.ndim == 3: @@ -135,8 +162,7 @@ def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None): else: n_samples = 1 - if mask == None: - mask = tensor.alloc(1., x.shape[0], 1) + assert mask is not None def _slice(_x, n, dim): if _x.ndim == 3: @@ -154,38 +180,46 @@ def _step(m_, x_, h_, c_): c = tensor.tanh(_slice(preact, 3, options['dim_proj'])) c = f * c_ + i * c - c = m_[:,None] * c + (1. - m_)[:,None] * c_ + c = m_[:, None] * c + (1. - m_)[:, None] * c_ h = o * tensor.tanh(c) - h = m_[:,None] * h + (1. - m_)[:,None] * h_ + h = m_[:, None] * h + (1. - m_)[:, None] * h_ return h, c - state_below = tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')] + state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) + + tparams[_p(prefix, 'b')]) - rval, updates = theano.scan(_step, + dim_proj = options['dim_proj'] + rval, updates = theano.scan(_step, sequences=[mask, state_below], - outputs_info = [tensor.alloc(0., n_samples, options['dim_proj']), - tensor.alloc(0., n_samples, options['dim_proj'])], + outputs_info=[tensor.alloc(0., n_samples, + dim_proj), + tensor.alloc(0., n_samples, + dim_proj)], name=_p(prefix, '_layers'), n_steps=nsteps) return rval[0] + def param_init_rconv(options, params, prefix='rconv'): - params[_p(prefix,'W')] = ortho_weight(options['dim_proj']) - params[_p(prefix,'U')] = ortho_weight(options['dim_proj']) - params[_p(prefix,'b')] = numpy.zeros((options['dim_proj'],)).astype('float32') - params[_p(prefix,'GW')] = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') - params[_p(prefix,'GU')] = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') - params[_p(prefix,'Gb')] = numpy.zeros((3,)).astype('float32') + params[_p(prefix, 'W')] = ortho_weight(options['dim_proj']) + params[_p(prefix, 'U')] = ortho_weight(options['dim_proj']) + b = numpy.zeros((options['dim_proj'],)).astype('float32') + params[_p(prefix, 'b')] = b + gw = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') + params[_p(prefix, 'GW')] = gw + gu = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') + params[_p(prefix, 'GU')] = gu + params[_p(prefix, 'Gb')] = numpy.zeros((3,)).astype('float32') return params + def rconv_layer(tparams, state_below, options, prefix='rconv', mask=None): nsteps = state_below.shape[0] - if mask == None: - mask = tensor.alloc(1., x.shape[0], 1) + assert mask is not None def _step(m_, p_): l_ = p_ @@ -198,93 +232,125 @@ def _step(m_, p_): newact = eval(options['activ'])(ps_+pl_+tparams[_p(prefix, 'b')]) # gater - gt_ = tensor.dot(ls_, tparams[_p(prefix, 'GU')]) +\ - tensor.dot(l_, tparams[_p(prefix, 'GW')]) + \ - tparams[_p(prefix, 'Gb')] + gt_ = (tensor.dot(ls_, tparams[_p(prefix, 'GU')]) + + tensor.dot(l_, tparams[_p(prefix, 'GW')]) + + tparams[_p(prefix, 'Gb')]) if l_.ndim == 3: gt_shp = gt_.shape - gt_ = gt_.reshape((gt_shp[0]*gt_shp[1],gt_shp[2])) + gt_ = gt_.reshape((gt_shp[0] * gt_shp[1], gt_shp[2])) gt_ = tensor.nnet.softmax(gt_) if l_.ndim == 3: - gt_ = gt_.reshape((gt_shp[0],gt_shp[1],gt_shp[2])) + gt_ = gt_.reshape((gt_shp[0], gt_shp[1], gt_shp[2])) if p_.ndim == 3: - gn = gt_[:,:,0].dimshuffle(0,1,'x') - gl = gt_[:,:,1].dimshuffle(0,1,'x') - gr = gt_[:,:,2].dimshuffle(0,1,'x') + gn = gt_[:, :, 0].dimshuffle(0, 1, 'x') + gl = gt_[:, :, 1].dimshuffle(0, 1, 'x') + gr = gt_[:, :, 2].dimshuffle(0, 1, 'x') else: - gn = gt_[:,0].dimshuffle(0,'x') - gl = gt_[:,1].dimshuffle(0,'x') - gr = gt_[:,2].dimshuffle(0,'x') + gn = gt_[:, 0].dimshuffle(0, 'x') + gl = gt_[:, 1].dimshuffle(0, 'x') + gr = gt_[:, 2].dimshuffle(0, 'x') act = newact * gn + ls_ * gl + l_ * gr if p_.ndim == 3: - m_ = m_.dimshuffle('x',0,'x') + m_ = m_.dimshuffle('x', 0, 'x') else: m_ = m_.dimshuffle('x', 0) return tensor.switch(m_, act, l_) rval, updates = theano.scan(_step, - sequences = [mask[1:]], - outputs_info = [state_below], - name='layer_%s'%prefix, - n_steps = nsteps-1) + sequences=[mask[1:]], + outputs_info=[state_below], + name='layer_%s' % prefix, + n_steps=nsteps-1) seqlens = tensor.cast(mask.sum(axis=0), 'int64')-1 roots = rval[-1] if state_below.ndim == 3: - def _grab_root(seqlen,one_sample,prev_sample): + def _grab_root(seqlen, one_sample, prev_sample): return one_sample[seqlen] + dim_proj = options['dim_proj'] roots, updates = theano.scan(_grab_root, - sequences = [seqlens, roots.dimshuffle(1,0,2)], - outputs_info = [tensor.alloc(0., options['dim_proj'])], - name='grab_root_%s'%prefix) + sequences=[seqlens, + roots.dimshuffle(1, 0, 2)], + outputs_info=[tensor.alloc(0., dim_proj)], + name='grab_root_%s' % prefix) else: - roots = roots[seqlens] # there should be only one, so it's fine. + roots = roots[seqlens] # there should be only one, so it's fine. return roots + def adadelta(lr, tparams, grads, x, mask, y, cost): - zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()] - running_up2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rup2'%k) for k, p in tparams.iteritems()] - running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2'%k) for k, p in tparams.iteritems()] + zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_grad' % k) + for k, p in tparams.iteritems()] + running_up2 = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_rup2' % k) + for k, p in tparams.iteritems()] + running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_rgrad2' % k) + for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] - rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] + rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) + for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rg2up) - - updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, running_up2, running_grads2)] - ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] + + updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg + for zg, ru2, rg2 in zip(zipped_grads, + running_up2, + running_grads2)] + ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) + for ru2, ud in zip(running_up2, updir)] param_up = [(p, p + ud) for p, ud in zip(itemlist(tparams), updir)] - f_update = theano.function([lr], [], updates=ru2up+param_up, on_unused_input='ignore') + f_update = theano.function([lr], [], updates=ru2up+param_up, + on_unused_input='ignore') return f_grad_shared, f_update + def rmsprop(lr, tparams, grads, x, mask, y, cost): - zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad'%k) for k, p in tparams.iteritems()] - running_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad'%k) for k, p in tparams.iteritems()] - running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_rgrad2'%k) for k, p in tparams.iteritems()] + zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_grad' % k) + for k, p in tparams.iteritems()] + running_grads = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_rgrad' % k) + for k, p in tparams.iteritems()] + running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_rgrad2' % k) + for k, p in tparams.iteritems()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] - rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] - - f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rgup+rg2up) - - updir = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_updir'%k) for k, p in tparams.iteritems()] - updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] - param_up = [(p, p + udn[1]) for p, udn in zip(itemlist(tparams), updir_new)] - f_update = theano.function([lr], [], updates=updir_new+param_up, on_unused_input='ignore') + rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) + for rg2, g in zip(running_grads2, grads)] + + f_grad_shared = theano.function([x, mask, y], cost, + updates=zgup + rgup + rg2up) + + updir = [theano.shared(p.get_value() * numpy.float32(0.), + name='%s_updir' % k) + for k, p in tparams.iteritems()] + updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) + for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, + running_grads2)] + param_up = [(p, p + udn[1]) + for p, udn in zip(itemlist(tparams), updir_new)] + f_update = theano.function([lr], [], updates=updir_new+param_up, + on_unused_input='ignore') return f_grad_shared, f_update + def sgd(lr, tparams, grads, x, mask, y, cost): - gshared = [theano.shared(p.get_value() * 0., name='%s_grad'%k) for k, p in tparams.iteritems()] + gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) + for k, p in tparams.iteritems()] gsup = [(gs, g) for gs, g in zip(gshared, grads)] f_grad_shared = theano.function([x, mask, y], cost, updates=gsup) @@ -294,6 +360,7 @@ def sgd(lr, tparams, grads, x, mask, y, cost): return f_grad_shared, f_update + def build_model(tparams, options): trng = RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) @@ -305,11 +372,15 @@ def build_model(tparams, options): n_timesteps = x.shape[0] n_samples = x.shape[1] - emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, n_samples, options['dim_proj']]) - proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix=options['encoder'], mask=mask) + emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, + n_samples, + options['dim_proj']]) + proj = get_layer(options['encoder'])[1](tparams, emb, options, + prefix=options['encoder'], + mask=mask) if options['encoder'] == 'lstm': - proj = (proj * mask[:,:,None]).sum(axis=0) - proj = proj / mask.sum(axis=0)[:,None] + proj = (proj * mask[:, :, None]).sum(axis=0) + proj = proj / mask.sum(axis=0)[:, None] if options['use_dropout']: proj = dropout_layer(proj, use_noise, trng) @@ -318,10 +389,11 @@ def build_model(tparams, options): f_pred_prob = theano.function([x, mask], pred) f_pred = theano.function([x, mask], pred.argmax(axis=1)) - cost = -tensor.log(pred[tensor.arange(n_samples), y]+1e-8).mean() + cost = -tensor.log(pred[tensor.arange(n_samples), y] + 1e-8).mean() return trng, use_noise, x, mask, y, f_pred_prob, f_pred, cost + def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): n_samples = len(data[0]) probs = numpy.zeros((n_samples, 2)).astype('float32') @@ -329,21 +401,26 @@ def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): n_done = 0 for _, valid_index in iterator: - x, mask, y = prepare_data([data[0][t] for t in valid_index], numpy.array(data[1])[valid_index], maxlen=None) - pred_probs = f_pred_prob(x,mask) - probs[valid_index,:] = pred_probs + x, mask, y = prepare_data([data[0][t] for t in valid_index], + numpy.array(data[1])[valid_index], + maxlen=None) + pred_probs = f_pred_prob(x, mask) + probs[valid_index, :] = pred_probs n_done += len(valid_index) if verbose: - print '%d/%d samples classified'%(n_done,n_samples) + print '%d/%d samples classified' % (n_done, n_samples) return probs + def pred_error(f_pred, prepare_data, data, iterator, verbose=False): valid_err = 0 for _, valid_index in iterator: - x, mask, y = prepare_data([data[0][t] for t in valid_index], numpy.array(data[1])[valid_index], maxlen=None) - preds = f_pred(x,mask) + x, mask, y = prepare_data([data[0][t] for t in valid_index], + numpy.array(data[1])[valid_index], + maxlen=None) + preds = f_pred(x, mask) targets = numpy.array(data[1])[valid_index] valid_err += (preds == targets).sum() valid_err = 1. - numpy.float32(valid_err) / len(data[0]) @@ -351,24 +428,24 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): return valid_err -def train(dim_proj=100, +def train(dim_proj=100, patience=10, - max_epochs = 5000, + max_epochs=5000, dispFreq=100, activ='lambda x: tensor.tanh(x)', - decay_c = 0., - lrate=0.01, + decay_c=0., + lrate=0.01, n_words=100000, data_sym=False, - optimizer='rmsprop', + optimizer='rmsprop', encoder='rconv', saveto='model.npz', - noise_std=0., + noise_std=0., validFreq=1000, - saveFreq=1000, # save the parameters after every saveFreq updates + saveFreq=1000, # save the parameters after every saveFreq updates maxlen=50, - batch_size = 16, - valid_batch_size = 16, + batch_size=16, + valid_batch_size=16, dataset='sentiment140', use_dropout=False): @@ -388,10 +465,8 @@ def train(dim_proj=100, params = init_params(model_options) tparams = init_tparams(params) - trng, use_noise, \ - x, mask, y, \ - f_pred_prob, f_pred, cost = \ - build_model(tparams, model_options) + (trng, use_noise, x, mask, + y, f_pred_prob, f_pred, cost) = build_model(tparams, model_options) if decay_c > 0.: decay_c = theano.shared(numpy.float32(decay_c), name='decay_c') @@ -406,13 +481,17 @@ def train(dim_proj=100, f_grad = theano.function([x, mask, y], grads) lr = tensor.scalar(name='lr') - f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, x, mask, y, cost) + f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, + x, mask, y, cost) print 'Optimization' - - kf_valid = get_minibatches_idx(len(valid[0]), len(valid[0])/valid_batch_size, shuffle=True) - kf_test = get_minibatches_idx(len(test[0]), len(test[0])/valid_batch_size, shuffle=True) + kf_valid = get_minibatches_idx(len(valid[0]), + len(valid[0]) / valid_batch_size, + shuffle=True) + kf_test = get_minibatches_idx(len(test[0]), + len(test[0]) / valid_batch_size, + shuffle=True) history_errs = [] best_p = None @@ -428,7 +507,8 @@ def train(dim_proj=100, for eidx in xrange(max_epochs): n_samples = 0 - kf = get_minibatches_idx(len(train[0]), len(train[0])/batch_size, shuffle=True) + kf = get_minibatches_idx(len(train[0]), len(train[0])/batch_size, + shuffle=True) for _, train_index in kf: n_samples += train_index.shape[0] @@ -436,9 +516,10 @@ def train(dim_proj=100, use_noise.set_value(1.) y = [train[1][t] for t in train_index] - x, mask, y = prepare_data([train[0][t] for t in train_index], y, maxlen=maxlen) + x, mask, y = prepare_data([train[0][t]for t in train_index], + y, maxlen=maxlen) - if x == None: + if x is None: print 'Minibatch with zero sample under length ', maxlen continue @@ -455,12 +536,12 @@ def train(dim_proj=100, if numpy.mod(uidx, saveFreq) == 0: print 'Saving...', - if best_p != None: + if best_p is not None: params = best_p else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) - pkl.dump(model_options, open('%s.pkl'%saveto, 'wb')) + pkl.dump(model_options, open('%s.pkl' % saveto, 'wb')) print 'Done' if numpy.mod(uidx, validFreq) == 0: @@ -471,26 +552,30 @@ def train(dim_proj=100, history_errs.append([valid_err, test_err]) - if uidx == 0 or valid_err <= numpy.array(history_errs)[:,0].min(): + if (uidx == 0 or + valid_err <= numpy.array(history_errs)[:, + 0].min()): + best_p = unzip(tparams) bad_counter = 0 - if eidx > patience and valid_err >= numpy.array(history_errs)[:-patience,0].min(): + if (eidx > patience and + valid_err >= numpy.array(history_errs)[:-patience, + 0].min()): bad_counter += 1 if bad_counter > patience: print 'Early Stop!' estop = True break - print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err + print ('Train ', train_err, 'Valid ', valid_err, + 'Test ', test_err) - print 'Seen %d samples'%n_samples + print 'Seen %d samples' % n_samples if estop: break - - - if best_p is not None: + if best_p is not None: zipp(best_p, tparams) use_noise.set_value(0.) @@ -498,26 +583,27 @@ def train(dim_proj=100, valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) - print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err params = copy.copy(best_p) - numpy.savez(saveto, zipped_params=best_p, train_err=train_err, - valid_err=valid_err, test_err=test_err, history_errs=history_errs, - **params) + numpy.savez(saveto, zipped_params=best_p, train_err=train_err, + valid_err=valid_err, test_err=test_err, + history_errs=history_errs, **params) return train_err, valid_err, test_err def main(job_id, params): - print 'Anything printed here will end up in the output directory for job #%d' % job_id + print ('Anything printed here will end up in the output directory' + 'for job #%d' % job_id) print params + use_dropout = True if params['use-dropout'][0] else False trainerr, validerr, testerr = train(saveto=params['model'][0], dim_proj=params['dim-proj'][0], n_words=params['n-words'][0], decay_c=params['decay-c'][0], lrate=params['learning-rate'][0], - optimizer=params['optimizer'][0], + optimizer=params['optimizer'][0], activ=params['activ'][0], encoder=params['encoder'][0], maxlen=600, @@ -526,8 +612,8 @@ def main(job_id, params): validFreq=10000, dispFreq=10, saveFreq=100000, - dataset='imdb', #'stanford', #'sentiment140', - use_dropout=True if params['use-dropout'][0] else False) + dataset='imdb', + use_dropout=use_dropout) return validerr if __name__ == '__main__': @@ -535,23 +621,9 @@ def main(job_id, params): 'model': ['model_lstm.npz'], 'encoder': ['lstm'], 'dim-proj': [128], - 'n-words': [10000], + 'n-words': [10000], 'optimizer': ['adadelta'], 'activ': ['lambda x: tensor.tanh(x)'], - 'decay-c': [0.], + 'decay-c': [0.], 'use-dropout': [1], 'learning-rate': [0.0001]}) - - - - - - - - - - - - - - From 4bb23cfa79cd70239cf6ffb971b2bae231e2c48e Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Fri, 19 Dec 2014 15:58:34 -0500 Subject: [PATCH 114/417] pep8 imdb.py --- code/imdb.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index f063ccea..f98c9601 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -9,11 +9,12 @@ import theano import theano.tensor as T + def prepare_data(seqs, labels, maxlen=None): # x: a list of sentences lengths = [len(s) for s in seqs] - if maxlen != None: + if maxlen is not None: new_seqs = [] new_labels = [] new_lengths = [] @@ -35,11 +36,12 @@ def prepare_data(seqs, labels, maxlen=None): x = numpy.zeros((maxlen, n_samples)).astype('int64') x_mask = numpy.zeros((maxlen, n_samples)).astype('float32') for idx, s in enumerate(seqs): - x[:lengths[idx],idx] = s - x_mask[:lengths[idx],idx] = 1. + x[:lengths[idx], idx] = s + x_mask[:lengths[idx], idx] = 1. return x, x_mask, labels + def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): ''' Loads the dataset @@ -88,5 +90,3 @@ def remove_unk(x): test = (test_set_x, test_set_y) return train, valid, test - - From f715347601c85dde6853277fb54ce7a819982d7b Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 19 Dec 2014 22:40:04 -0500 Subject: [PATCH 115/417] Fix mlp weight init formula. Reported by Alex Rothberg --- doc/mlp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index 4ee9fdd7..2a74aaad 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -239,8 +239,8 @@ are to conserve variance of the activation as well as variance of back-propagate This allows information to flow well upward and downward in the network and reduces discrepancies between layers. Under some assumptions, a compromise between these two constraints leads to the following -initialization: :math:`uniform[-\frac{6}{\sqrt{fan_{in}+fan_{out}}},\frac{6}{\sqrt{fan_{in}+fan_{out}}}]` -for tanh and :math:`uniform[-4*\frac{6}{\sqrt{fan_{in}+fan_{out}}},4*\frac{6}{\sqrt{fan_{in}+fan_{out}}}]` +initialization: :math:`uniform[-\frac{\sqrt{6}}{\sqrt{fan_{in}+fan_{out}}},\frac{\sqrt{6}}{\sqrt{fan_{in}+fan_{out}}}]` +for tanh and :math:`uniform[-4*\frac{\sqrt{6}}{\sqrt{fan_{in}+fan_{out}}},4*\frac{\sqrt{6}}{\sqrt{fan_{in}+fan_{out}}}]` for sigmoid. Where :math:`fan_{in}` is the number of inputs and :math:`fan_{out}` the number of hidden units. For mathematical considerations please refer to [Xavier10]_. From 47466300898c6d5cb870f6e4bdd8ebdbbfba39b0 Mon Sep 17 00:00:00 2001 From: diogo149 Date: Fri, 19 Dec 2014 23:44:15 -0800 Subject: [PATCH 116/417] edit year on intro it is not 2010 anymore --- doc/intro.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/intro.txt b/doc/intro.txt index 86390e20..a87f66e3 100644 --- a/doc/intro.txt +++ b/doc/intro.txt @@ -16,7 +16,7 @@ For more about deep learning algorithms, see for example: - The monograph or review paper `Learning Deep Architectures for AI `_ (Foundations & Trends in Machine Learning, 2009). - The ICML 2009 Workshop on Learning Feature Hierarchies `webpage `_ has a `list of references `_. - The LISA `public wiki `_ has a `reading list `_ and a `bibliography `_. - - Geoff Hinton has `readings `_ from last year's `NIPS tutorial `_. + - Geoff Hinton has `readings `_ from 2009's `NIPS tutorial `_. The tutorials presented here will introduce you to some of the most important deep learning algorithms and will also show you how to run them using Theano_. Theano is a python library that makes writing deep learning models easy, and gives the option of From aa67d35ddebfda47e7633dec069a86322dae3c67 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Sat, 20 Dec 2014 13:17:05 -0500 Subject: [PATCH 117/417] added a link to download.sh --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 9bc53e62..93a3a617 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -31,7 +31,7 @@ recurrent neural on the Large Movie Review Dataset dataset. While the dataset is public, in this tutorial we provide a copy of the dataset that has previously been preprocessed according to the needs of this LSTM implementation. You can download this preprocessed version of the dataset -using the script `download.sh` and uncompress it. +using the script `download.sh `_ and uncompress it. Papers ====== From 4bc39a2b3baaecf37b1e9547c4780fb7c112c21d Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Sat, 20 Dec 2014 18:06:51 -0500 Subject: [PATCH 118/417] early stop kicks in only after patience+1 validation runs --- code/lstm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index a520f08f..64ee6b6d 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -558,7 +558,7 @@ def train(dim_proj=100, best_p = unzip(tparams) bad_counter = 0 - if (eidx > patience and + if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): bad_counter += 1 From 7c23975008517bf6f136b5c079dc8f162c71e554 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Sun, 21 Dec 2014 12:54:49 -0500 Subject: [PATCH 119/417] trailing blank space removed --- code/lstm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 64ee6b6d..2b57d8be 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -558,7 +558,7 @@ def train(dim_proj=100, best_p = unzip(tparams) bad_counter = 0 - if (len(history_errs) > patience and + if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): bad_counter += 1 From 789f57d16b5ae4c8dffe01625bba93563c0ef5a1 Mon Sep 17 00:00:00 2001 From: Jon Gauthier Date: Tue, 23 Dec 2014 11:13:54 -0700 Subject: [PATCH 120/417] Minor typo fix in rbm.txt Great tutorials! Just a quick typo fix here.. --- doc/rbm.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/rbm.txt b/doc/rbm.txt index b3880ae8..a8079012 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -484,8 +484,8 @@ Notice that we modify the updates dictionary to increment the index of bit :math:`i`. This will result in bit :math:`i` cycling over all possible values :math:`\{0,1,...,N\}`, from one update to another. -Note that for CD training the cost-entropy cost between the input and the -reconstruction( the same as the one used for the de-noising autoencoder) is more reliable then the pseudo-loglikelihood. Here is the code we use to +Note that for CD training the cross-entropy cost between the input and the +reconstruction (the same as the one used for the de-noising autoencoder) is more reliable then the pseudo-loglikelihood. Here is the code we use to compute the pseudo-likelihood: .. literalinclude:: ../code/rbm.py From fc5467e2d4c66af6a39c33c8c6489cd035fb4a40 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Tue, 6 Jan 2015 12:56:58 -0500 Subject: [PATCH 121/417] Rename intro to index to avoid getting an error on the root page on readthedocs.org --- doc/contents.txt | 2 +- doc/{intro.txt => index.txt} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/{intro.txt => index.txt} (100%) diff --git a/doc/contents.txt b/doc/contents.txt index c21e7806..cd7281ba 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -9,7 +9,7 @@ Contents :maxdepth: 2 LICENSE - intro + index gettingstarted logreg mlp diff --git a/doc/intro.txt b/doc/index.txt similarity index 100% rename from doc/intro.txt rename to doc/index.txt From 1c08ba8f19f85a4f4fdb9febcdcaee0a2ac6858d Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 13:14:01 -0500 Subject: [PATCH 122/417] better loading of imdb dataset --- code/imdb.py | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index f98c9601..0aaf641f 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -42,7 +42,33 @@ def prepare_data(seqs, labels, maxlen=None): return x, x_mask, labels -def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): +def get_dataset_file(dataset, default_dataset, origin): + '''Look for it as if it was a full path, if not, try local file, + if not try in the data directory. + + Download dataset if it is not present + + ''' + data_dir, data_file = os.path.split(dataset) + if data_dir == "" and not os.path.isfile(dataset): + # Check if dataset is in the data directory. + new_path = os.path.join( + os.path.split(__file__)[0], + "..", + "data", + dataset + ) + if os.path.isfile(new_path) or data_file == default_dataset: + dataset = new_path + + if (not os.path.isfile(dataset)) and data_file == default_dataset: + import urllib + print 'Downloading data from %s' % origin + urllib.urlretrieve(origin, dataset) + return dataset + + +def load_data(path="imdb.pkl.gz", n_words=100000, valid_portion=0.1): ''' Loads the dataset :type dataset: string @@ -53,10 +79,12 @@ def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): # LOAD DATA # ############# - print '... loading data' - # Load the dataset - f = open(path, 'rb') + path = get_dataset_file( + path, "imdb.pkl.gz", + "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz") + + f = gzip.open(path, 'rb') train_set = cPickle.load(f) test_set = cPickle.load(f) f.close() From 859e9c3c49a03b868ce5eb94ad6a0165ce574d40 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 13:51:37 -0500 Subject: [PATCH 123/417] don't use eval anymore --- code/lstm.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 2b57d8be..24f1959a 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -107,14 +107,10 @@ def init_tparams(params): tparams[kk] = theano.shared(params[kk], name=kk) return tparams -layers = {'ff': ('param_init_fflayer', 'fflayer'), - 'rconv': ('param_init_rconv', 'rconv_layer'), - 'lstm': ('param_init_lstm', 'lstm_layer')} - def get_layer(name): fns = layers[name] - return (eval(fns[0]), eval(fns[1])) + return fns def param_init_fflayer(options, params, prefix='ff'): @@ -129,7 +125,7 @@ def param_init_fflayer(options, params, prefix='ff'): def fflayer(tparams, state_below, options, prefix='rconv', **kwargs): pre_act = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]) - return eval(options['activ'])(pre_act) + return options['activ'](pre_act) def ortho_weight(ndim): @@ -229,7 +225,7 @@ def _step(m_, p_): ls_ = ps_ ps_ = tensor.dot(ps_, tparams[_p(prefix, 'U')]) pl_ = tensor.dot(p_, tparams[_p(prefix, 'W')]) - newact = eval(options['activ'])(ps_+pl_+tparams[_p(prefix, 'b')]) + newact = options['activ'](ps_+pl_+tparams[_p(prefix, 'b')]) # gater gt_ = (tensor.dot(ls_, tparams[_p(prefix, 'GU')]) + @@ -284,6 +280,11 @@ def _grab_root(seqlen, one_sample, prev_sample): return roots +layers = {'ff': (param_init_fflayer, fflayer), + 'rconv': (param_init_rconv, rconv_layer), + 'lstm': (param_init_lstm, lstm_layer)} + + def adadelta(lr, tparams, grads, x, mask, y, cost): zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad' % k) @@ -432,21 +433,21 @@ def train(dim_proj=100, patience=10, max_epochs=5000, dispFreq=100, - activ='lambda x: tensor.tanh(x)', + activ=tensor.tanh, decay_c=0., lrate=0.01, n_words=100000, data_sym=False, - optimizer='rmsprop', - encoder='rconv', - saveto='model.npz', + optimizer=rmsprop, + encoder='lstm', + saveto='lstm_model.npz', noise_std=0., validFreq=1000, saveFreq=1000, # save the parameters after every saveFreq updates maxlen=50, batch_size=16, valid_batch_size=16, - dataset='sentiment140', + dataset='imdb', use_dropout=False): # Model options @@ -481,8 +482,8 @@ def train(dim_proj=100, f_grad = theano.function([x, mask, y], grads) lr = tensor.scalar(name='lr') - f_grad_shared, f_update = eval(optimizer)(lr, tparams, grads, - x, mask, y, cost) + f_grad_shared, f_update = optimizer(lr, tparams, grads, + x, mask, y, cost) print 'Optimization' @@ -618,12 +619,12 @@ def main(job_id, params): if __name__ == '__main__': main(0, { - 'model': ['model_lstm.npz'], + 'model': ['lstm_model.npz'], 'encoder': ['lstm'], 'dim-proj': [128], 'n-words': [10000], - 'optimizer': ['adadelta'], - 'activ': ['lambda x: tensor.tanh(x)'], + 'optimizer': [adadelta], # adadelta and rmsprop avail + 'activ': [tensor.tanh], # The activation function from Theano. 'decay-c': [0.], 'use-dropout': [1], 'learning-rate': [0.0001]}) From e57dd0371cd2e1d204660c807f8321b5892b6803 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 13:52:11 -0500 Subject: [PATCH 124/417] force floatX=float32, as otherwise there is problem. The learning rate 0.005 get cast to float64 --- code/lstm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/lstm.py b/code/lstm.py index 24f1959a..ca35616f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -618,6 +618,10 @@ def main(job_id, params): return validerr if __name__ == '__main__': + + # We must have floatX=float32 for this tutorial to work correctly. + theano.config.floatX = "float32" + main(0, { 'model': ['lstm_model.npz'], 'encoder': ['lstm'], From e85246f0899857f648e6eb46627b89a98391456f Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 15:18:29 -0500 Subject: [PATCH 125/417] lstm: add comments --- code/lstm.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index ca35616f..dedc0401 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -44,11 +44,17 @@ def get_dataset(name): def zipp(params, tparams): + """ + When we reload the model. Needed for the GPU stuff. + """ for kk, vv in params.iteritems(): tparams[kk].set_value(vv) def unzip(zipped): + """ + When we pickle the model. Needed for the GPU stuff. + """ new_params = OrderedDict() for kk, vv in zipped.iteritems(): new_params[kk] = vv.get_value() @@ -79,7 +85,6 @@ def init_params(options): randn = numpy.random.rand(options['n_words'], options['dim_proj']) params['Wemb'] = (0.01 * randn).astype('float32') - # rconv params = get_layer(options['encoder'])[0](options, params, prefix=options['encoder']) @@ -122,7 +127,7 @@ def param_init_fflayer(options, params, prefix='ff'): return params -def fflayer(tparams, state_below, options, prefix='rconv', **kwargs): +def fflayer(tparams, state_below, options, prefix='ff', **kwargs): pre_act = (tensor.dot(state_below, tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]) return options['activ'](pre_act) @@ -396,6 +401,9 @@ def build_model(tparams, options): def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): + """ If you want to use a trained model, this is useful to compute + the probabilities of new examples. + """ n_samples = len(data[0]) probs = numpy.zeros((n_samples, 2)).astype('float32') @@ -416,6 +424,11 @@ def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): def pred_error(f_pred, prepare_data, data, iterator, verbose=False): + """ + Just compute the error + f_pred: Theano fct computing the prediction + prepare_data: usual prepare_data for that dataset. + """ valid_err = 0 for _, valid_index in iterator: x, mask, y = prepare_data([data[0][t] for t in valid_index], @@ -430,19 +443,18 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): def train(dim_proj=100, - patience=10, + patience=10, # number of epoch to wait before early stop if no progress max_epochs=5000, - dispFreq=100, + dispFreq=100, # display to stdout the training progress every N updates activ=tensor.tanh, - decay_c=0., - lrate=0.01, - n_words=100000, - data_sym=False, - optimizer=rmsprop, - encoder='lstm', + decay_c=0., # weight decay for the classifier + lrate=0.01, # learning rate for sgd (not used for adadelta and rmsprop) + n_words=100000, # wocabulary size + optimizer=adadelta, + encoder='lstm',# can be removed must be lstm. saveto='lstm_model.npz', noise_std=0., - validFreq=1000, + validFreq=1000, # after 1000 saveFreq=1000, # save the parameters after every saveFreq updates maxlen=50, batch_size=16, @@ -478,7 +490,7 @@ def train(dim_proj=100, f_cost = theano.function([x, mask, y], cost) - grads = tensor.grad(cost, wrt=itemlist(tparams)) + grads = tensor.grad(cost, wrt=tparams.values()) f_grad = theano.function([x, mask, y], grads) lr = tensor.scalar(name='lr') @@ -627,8 +639,8 @@ def main(job_id, params): 'encoder': ['lstm'], 'dim-proj': [128], 'n-words': [10000], - 'optimizer': [adadelta], # adadelta and rmsprop avail + 'optimizer': [adadelta], # sgd, adadelta and rmsprop available 'activ': [tensor.tanh], # The activation function from Theano. - 'decay-c': [0.], - 'use-dropout': [1], + 'decay-c': [0.], # + 'use-dropout': [1], # if disable slightly faster, but worst test error. 'learning-rate': [0.0001]}) From 4eeee9825aa12a42f2f60c29699890668397c774 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 15:18:57 -0500 Subject: [PATCH 126/417] lstm: remove rconv code --- code/lstm.py | 85 ++-------------------------------------------------- 1 file changed, 2 insertions(+), 83 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index dedc0401..181b3578 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -203,90 +203,9 @@ def _step(m_, x_, h_, c_): return rval[0] -def param_init_rconv(options, params, prefix='rconv'): - params[_p(prefix, 'W')] = ortho_weight(options['dim_proj']) - params[_p(prefix, 'U')] = ortho_weight(options['dim_proj']) - b = numpy.zeros((options['dim_proj'],)).astype('float32') - params[_p(prefix, 'b')] = b - gw = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') - params[_p(prefix, 'GW')] = gw - gu = 0.01 * numpy.random.randn(options['dim_proj'], 3).astype('float32') - params[_p(prefix, 'GU')] = gu - params[_p(prefix, 'Gb')] = numpy.zeros((3,)).astype('float32') - - return params - - -def rconv_layer(tparams, state_below, options, prefix='rconv', mask=None): - nsteps = state_below.shape[0] - - assert mask is not None - - def _step(m_, p_): - l_ = p_ - # new activation - ps_ = tensor.zeros_like(p_) - ps_ = tensor.set_subtensor(ps_[1:], p_[:-1]) - ls_ = ps_ - ps_ = tensor.dot(ps_, tparams[_p(prefix, 'U')]) - pl_ = tensor.dot(p_, tparams[_p(prefix, 'W')]) - newact = options['activ'](ps_+pl_+tparams[_p(prefix, 'b')]) - - # gater - gt_ = (tensor.dot(ls_, tparams[_p(prefix, 'GU')]) + - tensor.dot(l_, tparams[_p(prefix, 'GW')]) + - tparams[_p(prefix, 'Gb')]) - if l_.ndim == 3: - gt_shp = gt_.shape - gt_ = gt_.reshape((gt_shp[0] * gt_shp[1], gt_shp[2])) - gt_ = tensor.nnet.softmax(gt_) - if l_.ndim == 3: - gt_ = gt_.reshape((gt_shp[0], gt_shp[1], gt_shp[2])) - - if p_.ndim == 3: - gn = gt_[:, :, 0].dimshuffle(0, 1, 'x') - gl = gt_[:, :, 1].dimshuffle(0, 1, 'x') - gr = gt_[:, :, 2].dimshuffle(0, 1, 'x') - else: - gn = gt_[:, 0].dimshuffle(0, 'x') - gl = gt_[:, 1].dimshuffle(0, 'x') - gr = gt_[:, 2].dimshuffle(0, 'x') - - act = newact * gn + ls_ * gl + l_ * gr - - if p_.ndim == 3: - m_ = m_.dimshuffle('x', 0, 'x') - else: - m_ = m_.dimshuffle('x', 0) - return tensor.switch(m_, act, l_) - - rval, updates = theano.scan(_step, - sequences=[mask[1:]], - outputs_info=[state_below], - name='layer_%s' % prefix, - n_steps=nsteps-1) - - seqlens = tensor.cast(mask.sum(axis=0), 'int64')-1 - roots = rval[-1] - - if state_below.ndim == 3: - def _grab_root(seqlen, one_sample, prev_sample): - return one_sample[seqlen] - - dim_proj = options['dim_proj'] - roots, updates = theano.scan(_grab_root, - sequences=[seqlens, - roots.dimshuffle(1, 0, 2)], - outputs_info=[tensor.alloc(0., dim_proj)], - name='grab_root_%s' % prefix) - else: - roots = roots[seqlens] # there should be only one, so it's fine. - - return roots - - +# ff: Feed Forward (normal neural net), only useful to put after lstm +# before the classifier. layers = {'ff': (param_init_fflayer, fflayer), - 'rconv': (param_init_rconv, rconv_layer), 'lstm': (param_init_lstm, lstm_layer)} From f52e89f1219f44e3102d4879926203790fea9f46 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Thu, 8 Jan 2015 15:48:21 -0500 Subject: [PATCH 127/417] title fixed --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 93a3a617..140e5c62 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -1,6 +1,6 @@ .. _lstm: -Recurrent Neural Networks with Word Embeddings +LSTM Network for Sentiment Analysis ********************************************** Summary From 6fe4fa020fc7236ae8c4c69f2d154198ae048591 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 8 Jan 2015 16:38:19 -0500 Subject: [PATCH 128/417] Code simplification. --- code/lstm.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 181b3578..f0361512 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -61,10 +61,6 @@ def unzip(zipped): return new_params -def itemlist(tparams): - return [vv for kk, vv in tparams.iteritems()] - - def dropout_layer(state_before, use_noise, trng): proj = tensor.switch(use_noise, (state_before * @@ -232,7 +228,7 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): running_grads2)] ru2up = [(ru2, 0.95 * ru2 + 0.05 * (ud ** 2)) for ru2, ud in zip(running_up2, updir)] - param_up = [(p, p + ud) for p, ud in zip(itemlist(tparams), updir)] + param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)] f_update = theano.function([lr], [], updates=ru2up+param_up, on_unused_input='ignore') @@ -266,7 +262,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] param_up = [(p, p + udn[1]) - for p, udn in zip(itemlist(tparams), updir_new)] + for p, udn in zip(tparams.values(), updir_new)] f_update = theano.function([lr], [], updates=updir_new+param_up, on_unused_input='ignore') @@ -280,7 +276,7 @@ def sgd(lr, tparams, grads, x, mask, y, cost): f_grad_shared = theano.function([x, mask, y], cost, updates=gsup) - pup = [(p, p - lr * g) for p, g in zip(itemlist(tparams), gshared)] + pup = [(p, p - lr * g) for p, g in zip(tparams.values(), gshared)] f_update = theano.function([lr], [], updates=pup) return f_grad_shared, f_update From 6b7d587a38c6ada08266ad51dcc336d819ffc0d9 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 12:00:37 -0500 Subject: [PATCH 129/417] Fix typo, add docstring, add timming, remove useless printing --- code/imdb.py | 9 +++++++++ code/lstm.py | 29 ++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index 0aaf641f..73e2d7b7 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -11,6 +11,15 @@ def prepare_data(seqs, labels, maxlen=None): + """Create the matrices from the datasets. + + This pad each sequence to the same lenght: the lenght of the + longuest sequence or maxlen. + + if maxlen is set, we will cut all sequence to this maximum + lenght. + + """ # x: a list of sentences lengths = [len(s) for s in seqs] diff --git a/code/lstm.py b/code/lstm.py index f0361512..e3c21f5e 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -1,16 +1,17 @@ ''' Build a tweet sentiment analyzer ''' -import theano -import theano.tensor as tensor -from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams - -import cPickle as pkl -import numpy +from collections import OrderedDict import copy +import cPickle as pkl import random +import sys +import time -from collections import OrderedDict +import numpy +import theano +import theano.tensor as tensor +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams import imdb @@ -364,7 +365,7 @@ def train(dim_proj=100, activ=tensor.tanh, decay_c=0., # weight decay for the classifier lrate=0.01, # learning rate for sgd (not used for adadelta and rmsprop) - n_words=100000, # wocabulary size + n_words=100000, # vocabulary size optimizer=adadelta, encoder='lstm',# can be removed must be lstm. saveto='lstm_model.npz', @@ -432,6 +433,7 @@ def train(dim_proj=100, uidx = 0 estop = False + start_time = time.clock() for eidx in xrange(max_epochs): n_samples = 0 @@ -502,9 +504,11 @@ def train(dim_proj=100, if estop: break - + end_time = time.clock() if best_p is not None: zipp(best_p, tparams) + else: + best_p = unzip(tparams) use_noise.set_value(0.) train_err = pred_error(f_pred, prepare_data, train, kf) @@ -518,12 +522,15 @@ def train(dim_proj=100, valid_err=valid_err, test_err=test_err, history_errs=history_errs, **params) + print 'The code run for %d epochs, with %f epochs/sec' % ( + uidx, 1. * uidx / (end_time - start_time)) + print >> sys.stderr, ('The code for file ' + + os.path.split(__file__)[1] + + ' ran for %.1fs' % ((end_time - start_time))) return train_err, valid_err, test_err def main(job_id, params): - print ('Anything printed here will end up in the output directory' - 'for job #%d' % job_id) print params use_dropout = True if params['use-dropout'][0] else False trainerr, validerr, testerr = train(saveto=params['model'][0], From 1d12bee18115e0a150d2ed92780cb3884b42ff88 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 13:38:08 -0500 Subject: [PATCH 130/417] use the not compressed version of imdb. This take 1s to load instead of 45s --- code/imdb.py | 10 +++++++--- code/lstm.py | 2 +- data/download.sh | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index 73e2d7b7..1bcc83bb 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -77,7 +77,7 @@ def get_dataset_file(dataset, default_dataset, origin): return dataset -def load_data(path="imdb.pkl.gz", n_words=100000, valid_portion=0.1): +def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): ''' Loads the dataset :type dataset: string @@ -91,9 +91,13 @@ def load_data(path="imdb.pkl.gz", n_words=100000, valid_portion=0.1): # Load the dataset path = get_dataset_file( path, "imdb.pkl.gz", - "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz") + "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl") + + if path.endswith(".gz"): + f = gzip.open(path, 'rb') + else: + f = open(path, 'rb') - f = gzip.open(path, 'rb') train_set = cPickle.load(f) test_set = cPickle.load(f) f.close() diff --git a/code/lstm.py b/code/lstm.py index e3c21f5e..c5e2bd98 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -471,7 +471,7 @@ def train(dim_proj=100, else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) - pkl.dump(model_options, open('%s.pkl' % saveto, 'wb')) + pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) print 'Done' if numpy.mod(uidx, validFreq) == 0: diff --git a/data/download.sh b/data/download.sh index 8a8e9a92..88e48e5a 100755 --- a/data/download.sh +++ b/data/download.sh @@ -15,7 +15,7 @@ fi $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz -$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip imdb.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" $DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz From f96d201b49a9cc4ff39a32531f1dc186abd6e9b1 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 16:41:24 -0500 Subject: [PATCH 131/417] remove import not used --- code/imdb.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index 1bcc83bb..c33884d6 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -1,13 +1,10 @@ import cPickle import gzip import os -import sys -import time import numpy import theano -import theano.tensor as T def prepare_data(seqs, labels, maxlen=None): From 9942cb826b59773549fa9f36be291ddc94facb1b Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 16:42:17 -0500 Subject: [PATCH 132/417] add name to fct --- code/lstm.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index c5e2bd98..23887e06 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -221,7 +221,8 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] - f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rg2up) + f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rg2up, + name='adadelta_f_grad_shared') updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg for zg, ru2, rg2 in zip(zipped_grads, @@ -232,7 +233,8 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)] f_update = theano.function([lr], [], updates=ru2up+param_up, - on_unused_input='ignore') + on_unused_input='ignore', + name='adadelta_f_update') return f_grad_shared, f_update @@ -254,7 +256,8 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): for rg2, g in zip(running_grads2, grads)] f_grad_shared = theano.function([x, mask, y], cost, - updates=zgup + rgup + rg2up) + updates=zgup + rgup + rg2up, + name='rmsprop_f_grad_shared') updir = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_updir' % k) @@ -265,7 +268,8 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): param_up = [(p, p + udn[1]) for p, udn in zip(tparams.values(), updir_new)] f_update = theano.function([lr], [], updates=updir_new+param_up, - on_unused_input='ignore') + on_unused_input='ignore', + name='rmsprop_f_update') return f_grad_shared, f_update @@ -275,10 +279,12 @@ def sgd(lr, tparams, grads, x, mask, y, cost): for k, p in tparams.iteritems()] gsup = [(gs, g) for gs, g in zip(gshared, grads)] - f_grad_shared = theano.function([x, mask, y], cost, updates=gsup) + f_grad_shared = theano.function([x, mask, y], cost, updates=gsup, + name='sgd_f_grad_shared') pup = [(p, p - lr * g) for p, g in zip(tparams.values(), gshared)] - f_update = theano.function([lr], [], updates=pup) + f_update = theano.function([lr], [], updates=pup, + name='sgd_f_update') return f_grad_shared, f_update @@ -308,8 +314,8 @@ def build_model(tparams, options): pred = tensor.nnet.softmax(tensor.dot(proj, tparams['U'])+tparams['b']) - f_pred_prob = theano.function([x, mask], pred) - f_pred = theano.function([x, mask], pred.argmax(axis=1)) + f_pred_prob = theano.function([x, mask], pred, name='f_pred_prob') + f_pred = theano.function([x, mask], pred.argmax(axis=1), name='f_pred') cost = -tensor.log(pred[tensor.arange(n_samples), y] + 1e-8).mean() @@ -404,10 +410,10 @@ def train(dim_proj=100, weight_decay *= decay_c cost += weight_decay - f_cost = theano.function([x, mask, y], cost) + f_cost = theano.function([x, mask, y], cost, name='f_cost') grads = tensor.grad(cost, wrt=tparams.values()) - f_grad = theano.function([x, mask, y], grads) + f_grad = theano.function([x, mask, y], grads, name='f_grad') lr = tensor.scalar(name='lr') f_grad_shared, f_update = optimizer(lr, tparams, grads, From 1e6bce295f5a90df5e22cb7168c6b308cb1b7a34 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 16:43:55 -0500 Subject: [PATCH 133/417] add comment --- code/lstm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 23887e06..3d43e3ca 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -368,11 +368,11 @@ def train(dim_proj=100, patience=10, # number of epoch to wait before early stop if no progress max_epochs=5000, dispFreq=100, # display to stdout the training progress every N updates - activ=tensor.tanh, + activ=tensor.tanh, # The activation function from Theano. decay_c=0., # weight decay for the classifier lrate=0.01, # learning rate for sgd (not used for adadelta and rmsprop) n_words=100000, # vocabulary size - optimizer=adadelta, + optimizer=adadelta, # sgd, adadelta and rmsprop available encoder='lstm',# can be removed must be lstm. saveto='lstm_model.npz', noise_std=0., @@ -382,7 +382,8 @@ def train(dim_proj=100, batch_size=16, valid_batch_size=16, dataset='imdb', - use_dropout=False): + use_dropout=False, # if False slightly faster, but worst test error + ): # Model options model_options = locals().copy() From 1b17e874e2e6eaa31344e48c09456303cd9a159a Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 16:45:27 -0500 Subject: [PATCH 134/417] pep8 printing --- code/lstm.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 3d43e3ca..48d012e9 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -372,11 +372,11 @@ def train(dim_proj=100, decay_c=0., # weight decay for the classifier lrate=0.01, # learning rate for sgd (not used for adadelta and rmsprop) n_words=100000, # vocabulary size - optimizer=adadelta, # sgd, adadelta and rmsprop available - encoder='lstm',# can be removed must be lstm. + optimizer=adadelta, # sgd, adadelta and rmsprop available + encoder='lstm', # can be removed must be lstm. saveto='lstm_model.npz', noise_std=0., - validFreq=1000, # after 1000 + validFreq=1000, # after 1000 saveFreq=1000, # save the parameters after every saveFreq updates maxlen=50, batch_size=16, @@ -529,11 +529,10 @@ def train(dim_proj=100, valid_err=valid_err, test_err=test_err, history_errs=history_errs, **params) - print 'The code run for %d epochs, with %f epochs/sec' % ( - uidx, 1. * uidx / (end_time - start_time)) - print >> sys.stderr, ('The code for file ' + - os.path.split(__file__)[1] + - ' ran for %.1fs' % ((end_time - start_time))) + print 'The code run for %d epochs, with %f sec/epochs' % ( + (eidx + 1), 1. * (eidx + 1) / (end_time - start_time)) + print >> sys.stderr, ('Training took %.1fs minutes' % + (end_time - start_time)) return train_err, valid_err, test_err From 64eeb12543a88f0f1ab2ea4abd4c584e562efce1 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 9 Jan 2015 16:48:42 -0500 Subject: [PATCH 135/417] code simplification --- code/lstm.py | 60 +++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 48d012e9..df3e380e 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -374,16 +374,16 @@ def train(dim_proj=100, n_words=100000, # vocabulary size optimizer=adadelta, # sgd, adadelta and rmsprop available encoder='lstm', # can be removed must be lstm. - saveto='lstm_model.npz', + saveto='lstm_model.npz', # The best model will be saved there noise_std=0., validFreq=1000, # after 1000 saveFreq=1000, # save the parameters after every saveFreq updates - maxlen=50, + maxlen=50, # longer sequence get ignored batch_size=16, valid_batch_size=16, dataset='imdb', use_dropout=False, # if False slightly faster, but worst test error - ): +): # Model options model_options = locals().copy() @@ -536,39 +536,31 @@ def train(dim_proj=100, return train_err, valid_err, test_err -def main(job_id, params): - print params - use_dropout = True if params['use-dropout'][0] else False - trainerr, validerr, testerr = train(saveto=params['model'][0], - dim_proj=params['dim-proj'][0], - n_words=params['n-words'][0], - decay_c=params['decay-c'][0], - lrate=params['learning-rate'][0], - optimizer=params['optimizer'][0], - activ=params['activ'][0], - encoder=params['encoder'][0], - maxlen=600, - batch_size=16, - valid_batch_size=16, - validFreq=10000, - dispFreq=10, - saveFreq=100000, - dataset='imdb', - use_dropout=use_dropout) - return validerr - if __name__ == '__main__': # We must have floatX=float32 for this tutorial to work correctly. theano.config.floatX = "float32" + theano.config.scan.allow_gc = False + + # See function train for all possible parameter and there definition. + trainerr, validerr, testerr = train( + saveto='lstm_model.npz', # The best model will be saved there + dim_proj=128, + n_words=10000, + decay_c=0, + lrate=0.0001, + optimizer=sgd, + activ=tensor.tanh, + encoder='lstm', + maxlen=100, # longer get ignored + batch_size=64, + valid_batch_size=64, + validFreq=10000, + dispFreq=10, + saveFreq=100000, + dataset='imdb', + use_dropout=True, + + max_epochs=2, + ) - main(0, { - 'model': ['lstm_model.npz'], - 'encoder': ['lstm'], - 'dim-proj': [128], - 'n-words': [10000], - 'optimizer': [adadelta], # sgd, adadelta and rmsprop available - 'activ': [tensor.tanh], # The activation function from Theano. - 'decay-c': [0.], # - 'use-dropout': [1], # if disable slightly faster, but worst test error. - 'learning-rate': [0.0001]}) From 86e5c4b49e5b6c9e2134fc1959339a2f314cfca3 Mon Sep 17 00:00:00 2001 From: Frederic Date: Sat, 10 Jan 2015 15:04:36 -0500 Subject: [PATCH 136/417] move sgd and comments --- code/lstm.py | 118 ++++++++++++++++++++++++++------------------------- 1 file changed, 61 insertions(+), 57 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index df3e380e..2bb845bb 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -206,6 +206,34 @@ def _step(m_, x_, h_, c_): 'lstm': (param_init_lstm, lstm_layer)} +def sgd(lr, tparams, grads, x, mask, y, cost): + """ Stochastic Gradient Descent + + :note: A more complicated version of sgd then needed. This is + done like that for adadelta and rmsprop. + + """ + # New set of shared variable that will contain the gradient + # for a mini-batch. + gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) + for k, p in tparams.iteritems()] + gsup = [(gs, g) for gs, g in zip(gshared, grads)] + + # Function that computes gradients for a mini-batch, but do not + # updates the weights. + f_grad_shared = theano.function([x, mask, y], cost, updates=gsup, + name='sgd_f_grad_shared') + + pup = [(p, p - lr * g) for p, g in zip(tparams.values(), gshared)] + + # Function that updates the weights from the previously computed + # gradient. + f_update = theano.function([lr], [], updates=pup, + name='sgd_f_update') + + return f_grad_shared, f_update + + def adadelta(lr, tparams, grads, x, mask, y, cost): zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), name='%s_grad' % k) @@ -274,21 +302,6 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): return f_grad_shared, f_update -def sgd(lr, tparams, grads, x, mask, y, cost): - gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) - for k, p in tparams.iteritems()] - gsup = [(gs, g) for gs, g in zip(gshared, grads)] - - f_grad_shared = theano.function([x, mask, y], cost, updates=gsup, - name='sgd_f_grad_shared') - - pup = [(p, p - lr * g) for p, g in zip(tparams.values(), gshared)] - f_update = theano.function([lr], [], updates=pup, - name='sgd_f_update') - - return f_grad_shared, f_update - - def build_model(tparams, options): trng = RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) @@ -319,7 +332,7 @@ def build_model(tparams, options): cost = -tensor.log(pred[tensor.arange(n_samples), y] + 1e-8).mean() - return trng, use_noise, x, mask, y, f_pred_prob, f_pred, cost + return use_noise, x, mask, y, f_pred_prob, f_pred, cost def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): @@ -364,25 +377,29 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): return valid_err -def train(dim_proj=100, - patience=10, # number of epoch to wait before early stop if no progress - max_epochs=5000, - dispFreq=100, # display to stdout the training progress every N updates - activ=tensor.tanh, # The activation function from Theano. - decay_c=0., # weight decay for the classifier - lrate=0.01, # learning rate for sgd (not used for adadelta and rmsprop) - n_words=100000, # vocabulary size - optimizer=adadelta, # sgd, adadelta and rmsprop available - encoder='lstm', # can be removed must be lstm. - saveto='lstm_model.npz', # The best model will be saved there - noise_std=0., - validFreq=1000, # after 1000 - saveFreq=1000, # save the parameters after every saveFreq updates - maxlen=50, # longer sequence get ignored - batch_size=16, - valid_batch_size=16, - dataset='imdb', - use_dropout=False, # if False slightly faster, but worst test error +def test_lstm( + dim_proj=128, # TODO: What is this + patience=10, # number of epoch to wait before early stop if no progress + max_epochs=5000, # The maximum number of epoch to run + dispFreq=10, # display to stdout the training progress every N updates + activ=tensor.tanh, # The activation function from Theano. + decay_c=0., # weight decay for the classifier applied to the U weights. + lrate=0.0001, # learning rate for sgd (not used for adadelta and rmsprop) + n_words=10000, # vocabulary size + optimizer=sgd, # sgd, adadelta and rmsprop available + encoder='lstm', # TODO: can be removed must be lstm. + saveto='lstm_model.npz', # The best model will be saved there + validFreq=10000, # after 1000 + saveFreq=100000, # save the parameters after every saveFreq updates + maxlen=100, # longer sequence get ignored + batch_size=64, + valid_batch_size=64, + dataset='imdb', + + # Parameter for extra option + noise_std=0., + use_dropout=False, # if False slightly faster, but worst test error + # TODO: This frequently need a bigger model. ): # Model options @@ -398,10 +415,17 @@ def train(dim_proj=100, model_options['ydim'] = ydim print 'Building model' + # This create the initial parameters as numpy ndarrays. + # Dict name (string) -> numpy ndarray params = init_params(model_options) + + # This create Theano Shared Variable from the parameters. + # Dict name (string) -> Theano Tensor Shared Variable + # params and tparams have different copy of the weights. tparams = init_tparams(params) - (trng, use_noise, x, mask, + # use_noise is for dropout + (use_noise, x, mask, y, f_pred_prob, f_pred, cost) = build_model(tparams, model_options) if decay_c > 0.: @@ -543,24 +567,4 @@ def train(dim_proj=100, theano.config.scan.allow_gc = False # See function train for all possible parameter and there definition. - trainerr, validerr, testerr = train( - saveto='lstm_model.npz', # The best model will be saved there - dim_proj=128, - n_words=10000, - decay_c=0, - lrate=0.0001, - optimizer=sgd, - activ=tensor.tanh, - encoder='lstm', - maxlen=100, # longer get ignored - batch_size=64, - valid_batch_size=64, - validFreq=10000, - dispFreq=10, - saveFreq=100000, - dataset='imdb', - use_dropout=True, - - max_epochs=2, - ) - + test_lstm(max_epochs=10) From c480d4eb1523e014ea8bb4579565fffdbd0b0583 Mon Sep 17 00:00:00 2001 From: Frederic Date: Sat, 10 Jan 2015 15:07:20 -0500 Subject: [PATCH 137/417] remove fflayers --- code/lstm.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 2bb845bb..c7f0c85f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -115,21 +115,6 @@ def get_layer(name): return fns -def param_init_fflayer(options, params, prefix='ff'): - weights = numpy.random.randn(options['dim_proj'], options['dim_proj']) - biases = numpy.zeros((options['dim_proj'], )) - params[_p(prefix, 'W')] = 0.01 * weights.astype('float32') - params[_p(prefix, 'b')] = biases.astype('float32') - - return params - - -def fflayer(tparams, state_below, options, prefix='ff', **kwargs): - pre_act = (tensor.dot(state_below, - tparams[_p(prefix, 'W')]) + tparams[_p(prefix, 'b')]) - return options['activ'](pre_act) - - def ortho_weight(ndim): W = numpy.random.randn(ndim, ndim) u, s, v = numpy.linalg.svd(W) @@ -202,8 +187,7 @@ def _step(m_, x_, h_, c_): # ff: Feed Forward (normal neural net), only useful to put after lstm # before the classifier. -layers = {'ff': (param_init_fflayer, fflayer), - 'lstm': (param_init_lstm, lstm_layer)} +layers = {'lstm': (param_init_lstm, lstm_layer)} def sgd(lr, tparams, grads, x, mask, y, cost): @@ -382,7 +366,6 @@ def test_lstm( patience=10, # number of epoch to wait before early stop if no progress max_epochs=5000, # The maximum number of epoch to run dispFreq=10, # display to stdout the training progress every N updates - activ=tensor.tanh, # The activation function from Theano. decay_c=0., # weight decay for the classifier applied to the U weights. lrate=0.0001, # learning rate for sgd (not used for adadelta and rmsprop) n_words=10000, # vocabulary size From 2dea44593ad7e315b584be4ce42d3eb4b1b7ff5c Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 10:58:29 -0500 Subject: [PATCH 138/417] Added better description of LSTMs in the LSTM tutorial --- doc/images/lstm.png | Bin 0 -> 13104 bytes doc/images/lstm_memorycell.png | Bin 0 -> 14362 bytes doc/lstm.txt | 200 ++++++++++++++++++++++++++++++--- 3 files changed, 182 insertions(+), 18 deletions(-) create mode 100644 doc/images/lstm.png create mode 100644 doc/images/lstm_memorycell.png diff --git a/doc/images/lstm.png b/doc/images/lstm.png new file mode 100644 index 0000000000000000000000000000000000000000..5620e3a0b3fd7824eaf281a7f051e50640fbcacc GIT binary patch literal 13104 zcmeHuWmH!Ex9vudZjh3eMoB@G?hrv5X%Gd2QlwM5LlF>_5Co*VyHrq=kZw^rr2DS@ zp8q}LjB(DrAMZUMj=m0GAK1^c_xi0h*PL@L!ftCQ6XDb0BM3sIs-mEcAQ;W?Cjb{4 zUa4PfE`h(Wt!^qSAZO_Bw1(^#2*Ql0D#+@(y;+@bH_;nBl|CK9u#Ts9I_D%qgmoT^ zll(iSsv_RaZ4WJ;9y}cNI-i~xs;sKQ>t0X3+^LBbwS8bkoc`**Ee}t41{cr67B4d^tM)m} z*wE0-c7%FM4SolSQFVp&KcY<)v_e*o_-Jrr2>mg@lB@r_1BO z^_N;7`9TEOv6+*J64ZkHF>&ZlL>C>#t1_=*GcQKw`lQ>9mg6}&Il=oYV0S+9vljik zwPpF>LGI2v`jsi~J<|l1{rs`a?72qvtWDh5p5TIlD-Q=h-rF9_|I*;Ow>=pmxP2fd zziOPZB!GNQd-CK7DFp>4H8r)Nv2j~R$9W14_2#$Yt~>=P2?;^P#R6htVrjAg*rAb; z?WWk0WaLHXYJe)!&vRH!_ygc5r>J1>5YUtIj=!sJA9zi_JWqg z-uP3*Z+zSY3mbcOvcWS+!d+;z{6WC4U)F>)0y1-_lU`f59z#sqf{EIiPIveRKI##S zjEtygXeim+^U;Yu4sCB&xNK5Qnyv9x&dO?5r|Mwg$0b5~G14jDGf8Yh8owcPUxwY? z-A|uBHUBIy7Aahv?TEm|!;`hPW(y`}2)}sEJ}oB)mxYBz<<_l^g&*%{R#&4^t~pqZ zl-^sIY!Huz=)iT-Tc^1XQeUs3&mJy1_g-CJpDLJ;_PN%3B^DkY{IipFUqPF1^j9Ab zla`c}(4Be5W@5!*o4L7){2u5_3(v!7U4Sa^7pJ8W-b!+CsilDhjt!PYh>XDwB(!;a6eSOy_oZ#whI zdUo^X&D2qY5mAiLuPN6OADI_h_}|Xeoy|`5;Td|Dkih=f&Q4KR_hs#^v6J@^QbFGz zr#`Hm;{BQXb)W0VJ3);-no;tjeVy-FBOH6-#_n$6z*NoCKY>9(K{VnnD>{!ZaXVWf z@k-8Qevot2w^Bq4C@I<5*)vpMaJ2UK>rDLq{hPzY#DvFg?$_rRbT}4UzrQNNIdyx> z>MN!o+q=6|(b3TvIXMeqV_xJ)hF+e2!l<{mcTiPT)u+#&bGPT#*ZCo9KA0K&S2xJS zl&UtiI+&MOTvnDh@D0htzp+WjOxu&dYcRdIcxR37B4TK0m?j@WT1Le7r= zNGpwHa~uuC9yM4Ql0OK*P{E#CS_+4{l*QFri^wZ4r(9fIgfiBvs(3hvPt9kNQyv9( zS8$kbP{bHa!W2Ez7J`XDmP99-`_K|lU2cW|BDQ<%OTT{y_h)O)0rK5w3TiT~bNTf3 ztD>XhhR)F7;OUPKy+Z$1w<;-{vY7;EaZ)fcMnj(K*NzEENikO0Pqf0b4BBDh5V!9v z55>KFscS!xbu;nd_|!z5D{sZY(GjLhO}`Tapm0F{ZQLbg)@#?OnS76(vhHOBS4}D_ zDY1%+UtIoKkoV+ZF6P{V)tT)WXWEF=pmUQ!=GT4ZzVnP2@fxPJ&Uo;+1y4o-@hFPj zx9&$WO44y^yzxs+q}F(QBM45n_3Kw6coc&oaqmOt$K%yxZ6PGek00kBik|iKs3Wf# zDnuM6=}(UKLYkVS=|mm|0(`W0b;)OGLnW#;nEm=$8VQn-=6VJc?LdbNRITUGj+ z!z5*u!xYtAXB35ohDPg`FKS}q;t@$nNjz04$;sSu^726q4Gn0~(cEy(`Sk4@6`}yY z5>BLj3}(dF>o3(lTjqdt=Dd9QvO1%8A3b_Bv$n=(b@#5wXri87*=(h^msjB1w{MFpE2#<# z3t5GPv@4%iep5~g3c=^2{dv_|`SdXouln0$ddyF&H>m0H$R#lYVEqG&G zT0x`@8f1BEYwP#gXOEoM#%Vh`I#kruTA?yY>FFb5V@b2Rb?EmoxEK}| zrlPK{;N&D|RBr3VE65bZ9PGS?97QTN=$>?A)JM|@>f2a!QJ_k*oimED}+Rd8@eDe6Js;aoe z#K91vzN{p6C|5(njU|$nL<|*c>+$jNjnPL%mcM(F)e_~bzwPr}Wxe}k{>ugH`}eK7 zUtK|8+@wT$YWgvs3Eksj{@hAQrMk9P~wc zahLq@z}gEoN}Q|{RtPz5Sh#=Ut5?BranMH=AzxrvmjJyC=eac*&yz2bz%{ri#JH|c z77R2#=;wI(@+H@8vbcl0aIODR^tSzEeHipiOapl~9Bk&e*3Ps$m!^3JTirTx4-CAu|6^ssDNT|4mHtU-J3oWB^be1E{GbrYF65L!X(Q{T|Rx zEzyb+`t|opJ7ZwE!kOf9xLFa*9^>s{6eU(2_$c!ge72<9lPYZv1o?RXFK%q4_S+lJ zivNvP2kop+nJ3=rrDlVVIjwJ?Ylg(TRKY3ftJ& zn39r`OE=y9b8l}*R#sNuJ<>(O8053{d2H+q;7x{2KHgdPmWPT$-oL-}{P}aroF0jh z(a{Y30wcJeOkqM9p*uf+{zS<#z!lU{jy?b7%k4l$qk#qTG!+xz<8QR!Ff~t1FaT?| z9WG`8hU>UF+wo+10!jh8Ku~Hb(>u?#>p;if18<>`_K^Sp=CP|FH@dY$7^lAt7>TAHORUwFxm})s38;um1h} z_lu-(^|NLDU-R=UH*VDS9RlP7iEXKO-}+u;E}d^uLk{TE26QHJZK7`OO9JoGGhrd2 zV(%jttn=qBXIg^{ORUbz$;lPlPu#%5!kPvikx-cm;sQ{TSxk)P=BvxHW0iI&hrN8K zR3WeT*jPtY5G@>N{2i01*BqUwX zv&cwE{f_q5rTc3~`+f=k`Td(!K!6lp=mLr;=zHn~z-7_whtWIEuN8&IdP&%_6>DZ; zVZZ$u68io7ZI(dN#>PfgHa3NZe21yV*4|!CTo#o4x_Kq>1M|(%<=O1lluj5k)7O7h zTv2gBOG|615=5Prk&#jEdSDKHPQhl|t4J^lkDAI&$`mmfLxHcH`0Jy@-pqC(n% zT6UlKI6FHhj9t;GbFtgp9dj_KiOdPU0U}S`4327dZ6X3B#q{CM@^F=dd4jSpQVe+Q z=I-7C*;Z3ivy^({*-{#$7)TiXB52iljRED=)imzz?!eeOBKR9|5izfQ$E;OIu=h6; z`Ie-`LHxl5pL%)*mV~d1i_Z-jm|q#%{b}rgfi#}%8Y8|;i=rSr$F(d+PHp=#@ha^n zKHB%+H?@24H~B?;{4^vTh{uGZ31C2yPn#4(I_hLT|2o-9J!>z2&`-+5M8bjv$tmcy zzri!~_0uOAA0H_oR=V?lyQUw45Rfbks^K5;x!FX}+?VC3|6@!JQ z7Y_~&M!nd|;l#94LPBtm9L;{SnkVNaJ+^bUlR|=m&fjoe@O$}^@>z=L^23`idcAg* z^g!_G8x?)*Rkh}VdUXfNsqE%feFJH0Ys-79nhs3Z$H!;C*Y|X5VD)s_{H!%YF@lte z3Yew`0Z5S`;AMfNsE&K^;6c_@+-9;ot+a45G{5Aag#}x$gZV_n56Gl`omrjBYHUhM z1e8~{{ZkME3j?_?4qx6)?s9nfBQY{EQq2xXAtq98KS75~|2rG!3j}Tu_wF6Bfq?;f z-%Ij&`UQ?qNNTHk-Wsg(Ga@oV1T8HsNy0Y7zrVeWkn%p%tuq5rSz1@wELMe6+xip5oBg&rrLdrgPek*yMUUBDF(EM<;HZ& zlBnI!!4R+>W+9V+F7XUCIxlKB?B3jFgMM3PVQJ}qr|iDuvxZ!qY^&aPk~ESYS9e!N z0-&R}b#`JK6q>XEk)58O4-$Fw6F~sq;Fx1DFtM30UnT?*xBolE0X_!P+m~MA?0Abt z_&y=3bSv$~toGI>hD+}epqG%6B1n>y_YF~p$w*KZbI>V3IJLrO7C)I+tFEcBJltMf zT76JfD5ah%#Q=%lzA@8AcJX2avz206KcMXXQgbwOUS_=mBQX+yTBAcDi@lO z!HS~6%-r0im6esiii!#lJs0BQ;#k?)6&ohaeNWgws3)_6xZ%jp&sP9t*^#OGq8;Q$ z`sUqpgM(?GZNwd?{Xn4Cp)ddmaRC+3T=1)E4b*B!wbNYSix)47OH0YWef!4B!J+i) z*Du}CLP)^FgL2T|$;qr&uj&qujYR<44o{a4313-xj3EO&T(^dQp9X7Di}M?&VvLNFPmFxUyG6~o2Hr=+b-4)#sZ^t44_U?7(%qiQHn zC+r_}t`2M*9IS$ZWI&c}e`K7WbpON9^G-75T1zY|$~8Mp_`7?P1Tls0_uQ@}WEC;P z{EzmW5DzpR&=s8lI3(ro#7XK(?2e-+BS*$6e61WL~mIG~u*EEe==e^e~iyvxdBH}hN-0Y!7My4B_>!0XCX z6BB>aaVb#ew*bY^KCqxW<(Srqf_Xs(6d#^$|KNbzlrdQp*t`CdCqfw+85>9IO$K$} zK$HtU9N@y#*Aq@pPv76{V%pr8k2`&La8?<4kh3_o;`}+t|>$*-1{^qmgpDj~U z<}+#)!cdYB$@eB>i~ZRbJ}NPuo$fUaX7{eHti;C0Ki`}5Bmvpo1*GG)k{3p*8Hhn8psXUoa?&V9!iF<|FI#@plEh3M!FD;oB?TI zcyLJ?gcBSw#QtTs>ta@MU7Ko(0RXiGk-optMzRTT_^q;En~Pl-t$scR$yNH6DjgF>!8yA(<=gdlzWYa+jb_$Op`@`Yhc;jr zpbpO?nIAr&J*G2+$tTD0p6-1xWPpCQfKMyvx%=4A$cV?3oR)+Uqpysci%Z7DgdPaA zfbDl?FlMzsZeaTT8>(rY*houDw|8`K8<6S=xoudyx$z9YE1E&x!h)I4yfNQd)mj10 zz5;wEfKJqb6ViAm5jH$*_@XXOky;~F%9IbmNE!$@E=386%gV4n5O+v3rd03Ub+$1q4Rz9=(P@viAnyFH1iJ3VNrZmWZAoFx!Xof~bfsrM^dg)RL(Aj*W z3Vb+ggW9Xwq%p{1Zg_Y&DK#~Yxw$!D%nYP=UkKBNE8^kq&TWeQpryZ`LOGh=KQE8h z%iG&&V_G)rChPf44jPXPFn6q41MzJ8KM=j*GYyA^#bp|=#dYP1B1j&`t>3p$Z(1!; zzSjcSj@w++wT&;lMHXPQNO)gin#7XVt9!Ogg*wP!8(Kn&4c9#7Y+DxuVPrdAE!fr@ zcpe7_ZJ2<=N-{Etkg#w9Ul<|O<2=fNp`@XcJ_oi0a|bm3S=~Z6vNcnGFKP;Z&(XT{ zc&w6$f>T35M#fL|R+pkkSBizoD)d%VN`UGiW|Y8%LWjdQvibHV*BOT^xiBY(%X#@H z5j0UT$7w{r$c#u?SvjklRdSAunFYK$;397G2?hedcG);Ng^%Ao$FTJQcf;}5Co+_u zj*VRucU{x1W$P2hPHX?dk+ZY0A#-xL18?Xb8j1@yh^Hrssq&%s%NKkYexMwIL-SoE zw8Pn=rc>?O%Y78sX_+XC0QxfnM79|i2#Tl!c?Oz`rRuFv;hRv|>5w#Fnt05vugAb6 zPQ%4=fZ|R|1Ke;pk`{>CZ(uQS>K%ah0x+cjuL|YAYKix>zrB+raCLPxyt7GOdp%Dl zn;3Z0=A`GkC1{E_jzffk2>a$fRz>ra`RoPG9QLmphp8s0wu49939M^6JVw#)it6yV(Sc+FF2?SbVK19az#&a@QoQLHs?c3#BA#g(zL z%78cp)A(_K>-31`sx7INl~u9r4^}Ujve{17*T4gdn4P`<2!bq16L4aVYcQLNFiNYUE0`y9+uL&s?ba~McS{n2g9`cvgz9{4nWe}5W5 zD=Z)A3ox_^h6^wlG3e4u@g9O~L1}NQ&tce%4c50e*V~$KJnk15S9Jo1pI%)h;^*f_ zM{;V3Y?9b%WMpy{J4=J}969rmM4bW3e!4mkeV?tb*Pw^vbB^oLV18G5bNwkcs=h%0 zppU0<`;j$h`w(!ut7~g{O!1Tb;h&d8L@t2*T5^qupAkeZ=!?qQsi@$hQ!}WV=Aj{j z#nPPTfaxI47^$d^Ipu4$vNaf>`!e(KMN_Qt1fIRV@G>^m50vfuv@{t>*(*&JQ4~dI zag8r;Ro6Wl;?t;pWN}=nC!?fdz zUiM%S;)4$AioV9SK-v(^b;bM%tUGLR^v1!h#_X99;j4(Pw{r|g$mUW(_sw>lPr4rL)tZ?qx<{`g$qGor9b?{@+xh*=g_9^ZXnwDUjV1WG6tG)cb>yiu+)FKWvn*TsGgHzeu zFbR{xLG{daG5IP0QcW)}cjLK=G$UT;Sg^>+$w8)e;5}sT&`Gma6Pqb?q4673(bPnd zuh{G0$>XU;h6rkYC5Q>+4F-uFO9_V?#vO2EK@irFCMnPo_`7Da~=Og{S``bEVl$cRl&Xnf~%mtpmDatbguaEh!9kZ z@?!iaT+;@R3xF32E-saQS)_|!0UZQkkQGTMqSlG$I`2+xtEELYJ39-}CjfE>gfD(Z zd60*jyJa9(CxI_Or3fl1B-Qtn77o5+FwY=?uWLH<`Z)<;6VPHMFPk^g19$@^Qc(Ev zSMM?c6sJ|tE2Pn(z z%3|JdH|oIloDydO=VRgJy$M$uLF{}^4{X3w;5H4Pq!)8UV7dv~R)`fOFwDugP>Kz* zI}f7Eb?DfbQ6@V)GQvI5Fu7T%Jpt-TaJi&4Og)$9)vH(cCD$;xPG7MPX+-D$2990EBoeP@=JVM0e^(&W=HzI*i~cV8T8c5runZcb-$ z@6(2unTmcb{ZVPO=ZiOFJ}@hxk#G}069LTJw0qsJs$JI^QRV|Wm>dTFj!sS(2q<{{ z!xI>kwsv*lfJ=~9Ndm$L2F)|5jk2?IrRp+(H|R8QFcf8F{q3dc7wNkQXd~)e*Jt5K z7GU-*YgS&gIGpG*P7nm+Al(sSnZV#+sZCc&Nl8VJE*X-qnA>39^5=ig+3(o}pDi=7v?Jrzr0AptOjzlG{>VGyBb*#4JzEf&r#pdkOzqO^CuOwxjZMP|DzqRGC6+-6E-GLjRK+Co}}*T z?iaEBj(`Oo4D7hpWi=e~oRyPPd9k#o59-ws#_e9)Ka@9sohI^`7F&NNZ&-2gj`7xE z+T)=0gZAGAr-1gnhFuByL|UweF~g9}J4j<_p6ZqsbSsK*uqkp&fpKRG$MFkT}J5<430(%`U1(TVikk%rgd=sTd&1RoCzAkFYU zuo}?y;K{>J;K111+gCfzoI_qh=GegYIW1drk*`?_H64 z>y)dPM+M^PONC@(Q$c?I^vX&kyl+ce+g$RIVEgd3c_)W>jQ!4w4$Xsu7XW`dLAatT zJE*y!udk0D7}#FUF2f8EuG7hxMExh_Mc8vD~Z7P zaM9w-kugi-e5cK0{Kt_ZNVW|jRQ zBn6Z{fa|bVAP4p}jEXBp&7LW*=~WDEb4|TuV`tAj1F8EH{kpC$rHQHO&S;TTm(Q(6 z&IIef82i;gGTws+0J|?~8dVn=7z(8Bm4K=B*4|hj#EF>8O44C3SVO3aP>QAx11E*{ z*EhlF#J^H!$4?QN7aX?+&M>t%NBBM_gDsqITt)0qzlm+;d-4os1U!FZtEHg00C)+} zjW~UGUDP?813yDV%2Zzqs2x-ZW`bIWs`E2zh@IQUG{(7e=LDYoQXyoxq01R_NjGS2 zesOKGAp)`yikbm+m@Mwyi#I1yDk?4}#UZBm192gHDEw>qT8ERv%lF4H?@AJPwNL1t z^jN+E$(8}Ul19vt8QqApvx|aaV1;Q238O?a>az5Ez@CP{qo0@G+UvN0^Z8GXXdF4# z510T&DE8QS2+hy4S90%-MwN~F-6_hh1}*IAAOdOu&)pTC#D;_4DPT!rqApWrCL0P$ za5AWC0?y0^446=_7`rg3dWlWOL&hVaNY58u6oh1CTDShMYn$5V(+YnNvu<@d$qB_fjq@JTTNoN9$sUP=MJyp9 zLAblt0F?<-MA}ULr=l>ggIz?Ld};ZZ7vL6vM*7C=9k)1K=UOD&bcqOUyi(cktuXds zU`AKo1b*=G9MhSMtu5Mu2!9rF_ujcrh_>?aCf_NG6K)X=;GRT3scD-Uy^rXeT}ZP! zPXOeuN~Gv9PtZt?0gd<%%H4c1(Tw&hURenZMwD{RS`iK4EWU8 z`)Iu*J(1Rl3Da`hA9D3)r$<&_UyHCZGsD7ds@LDJLh0w}&O)CNO1V(5ER7ISSkfl- zJ=saz*xsf@H}b$581gWNm8%SJ;`R7EH#?|1p7u%_^gmH$sio(<0DCiriyq^cu)6o3 zmJ5O6GM?$I1++-0_FcLjKtOD~7g+6Xjlka4`!;fKNG5Xk|FGif6IGeV_JX(>#%U&XE<^}A%7~ATw;7Et zD--rP*%{(VoTzrf1SIM;xx}q$Fc0_hRzdv-J>=Da{U z>3eo!GpY}i9X4$gLHTBzcxq@6^O@ELg4$Tlu-5kD>%xo6ovt2=?c=me9fVDGbQc-c zC{kf!4Aclr1%ge$^@j}wzy$CSD;RjskAnZrjf8^DvzU2bNWwJwMR+qT9A-L8uH{`^ zh9qtcyxV@&;rX39mjZ!SJ+_3r2j{Xh*F7SyCR*z`x#Pj1kCFNF4otp%8x_*wGm6i73 z^~yiUD`NSRzwH#tDYaGum?+V0JX+y@VYX&S=I5up_m4MK<-BT614sp*=$JnmtjIWQry)-B7`EW^`( z-9JVh&l!tIK&?altJ?KG?9YD2xfk_Zhcf8|dTj{Yl`gAc7F1&+bJ;Jwp!4Xfo0RkN zS8e*JR+H-{N$$VBKYK7`pNM7V`^5uSe09jukXygZtf+g!dRo-7=nD%z6HX9(wfFRW zidDXj!C?9gir1LoD;l4W8~OA3{~zCGSbU7TEIrqia`(XleEI0&JMXZwSV%|&*g5Nh z@KD9}v$9HQ;&iF0sk&umr`6Tf*Y=i*IOr7Fa&vQKwa~P;=$U!xqlu+1(mf7}rJ|t; zj*TTBd*eh{aH~pNB*IWiN~+jlPMP{+5ueFNX^bpO3ZEec2UYE~M2Vt)IsXQJmT4Ok z)J|RkYbzJ+PbccEd2FzqqpqpxA56y9G%!FB7Z+#e|nES2nY%Dj5e!Jou$7s-4w94wdMcp8O2EX z1HZL3yJQI;LzB7`QJ2Alde%2)>=!S-Er0NNYH2An?((gszCKd8iT9$bb$%Hc%mw>K zx9D&0ga#$DI%J(7j@p0~D0W;>H#IY3zHmWixn{J=Hq&KwqS|$xIU_T3dUn=NBU_n? zi3!VNXQib#^G0v!J;jP4xFr3fM~|SM-WK0&UR(7!KG)LHvbM9+2DfqLK_B|s;nwHj zvioNrKYq-=5&P^J)0Hd4Y|6>kH8siW>g(+f_nh+eD+t6qe!gFyZR^d|$QCIZ!lZHP z>FEtOc!}bkr4Pd=V@;O{r4)Mj6)!qEy4YpaF!`DrX=P=l2%VzWPk4Zfk{Gl<-kCF$ zBidVG&9q~36h=zw zajS#Cw_caVpq!2`EsEh7%{(ZHrj8E$%o~x~mJEDmMCa+~N}QIo!-pJy43pfwd)LDH z!Gj0LR!_b28yGP$Hs&`>OH3r-=H_;Fch9J-HA&P`=Vn&MaIjd;z-_#=tE=s#dVEvC&iOVjJ$HVPo09zYVWk+um-;PV*j-3tqJS-uWDP zqm;BXkJF;2kg)K(4<87m@NjT(GfGR<=d5Wydy_=*>bjDs6^xCI*{zHT<1zJ{)nDS^ zu>bi(uezpYp*$smPIRVf()+V{_lFOC<_<0{H|*_su(7cnM=H6}jPq^dPM^D)ZaGvq zTxp%8?_p?|I^ni8&dqG$E>v_AQ$fpXidSgVSh$W~|MjaWIXU^A`JNXd)@*1rj!vP0 zq`rQl{{Dw&90Gf?7|c{(mZF@3Lf@MLiOJTCMU8-+m2t~0&U^Rn>EEvqKUiy~tuoS} z_d*T>eHU`iW}gxRZYEu2x^#(vO(oSoKAwt~k1y)!Q`FhBXFC%G6z7&b54VS?d5m#b zRZkI74 zL+ABc+F&`g``xK6gxqz$3!k%D#Klwff^yhrT?9y@Jn65wruSBQ{(6Xd>x*?F(VCLk zLcF>)G{}^UmNrB=MdZb;DjQcQ_`bfrr>UtCzP>1^rmLfzrsvV}QqcUicjC15pyo(P zN$u8WWMbpvXZ_BKzkPg?Fcx2)H8OyTq~n!Qqqm&AyrhBx0lk>Jf8)V7?gYO3XTE<* z%HFxml&DV0Yly7;*1~5V*R?yH&-qSKQc|vMY^aN9$3!J2h9oB^n`yN_45uO?At~>6 zm64T&R(P_?W?X7%uz*5oh(EyJ{|*ckbJ;AsAy;_Oz}LZoTd1D&jHRv=(Qt}3MvIU4 ze|)(~g3M7~;jFBz-TleqGcGPJy88M}Pzp9SHm;kW1P5#|`%~{ua2iw-!}uvKEtP`i zL`+W~u{qa0BpisAA1E`5Mr%ndy5IaSBVkDNBlMZIFZa`s1uHHq^Mn3>O-xLnp}pRs zPJ$30zc;a>(lh*7y@hC71YPq-?na_>=gyVePSV4t>m+@NM&paRtn@GKUOj0rD}%wX z-XE*Bm%DkhCCRoS_&MMGrluzBc&^(`sbU_g1_ohQtw+-H^NFCx3)#=$KxI!YFNZG; z6`2gY)|&rP#)b?m*u4QESy=TruG=Ad#3^O1dr;RNXN#@ypNWv>Ja?@vcqzRzvFt!wXPi&@`z6@o=#Ey&2ghGF=vY0m5wG=}z$j&~&{Z9VlKE(Kod8XD8zzMWK2Q6Z)ZnCM-*LP(iVXS6Waxj6?wV!D5b;q&fZ~X6N8Y?aYZGiI_p1wYVRK# z-@pH)@OJG?dlbv*F55+Dn^E$jK1bIAiD(}G_+gcm`7}&BU-uCl%dPp|6L9?I2lA+J ziKu;{b2N8#ok2Z)_AH>UPrcl7NO|dMXXj#7r?7;CQRb(Pj@&({S5eDrMJjR7qjrD& zde_vHVfbq?hY0nXPN|q#Nig%wQJW@_4M$KjEp?@7P7-docDh&3yi!s32m*QPzwB@PNw#8nZ|-+ zo|ePIK^JXg1al;)9bI8FPRk-2_BM!w1%(21(PmjxAGo{gie_bP8Xule&04}3jY8outDA8@| z=IT1tpQ|yjPlS4MmZ7z%@#yu^N`HrH8_90>w;L#nUr8b`4oIRppkKaAm-L&On|tEC zJo2Ktn!2N-<0%>+iVWrZ;PWdc7M2rgYHHBtuQz%h-nn-#pswzksj2DPj~@wxgM&-I z4zMXIDG|YY18N-RX4a;gsfFwq^9u^1Vq*M0B?;z@i!Lo!O}4^JASNgGhmAITJ;cqS{}kQ8tUCE$V#0F%bhbd9@H#VnAr#~2tSudT0>3=R$gDpZGY91O=4fesZ7 z<}XL(&1>uZ8uVW8xqEmJ0v6)Dd^sRHJ3HZ;Td95{fCL_w6@%&d`HPyDo(MdQH3l(v&E5TdLQ6}_>91eWFoM_s{49;@;4uLTld-bOhs`1e2GI2Gouq$2KrqZ6 z_vLe_f=G$Em|5MYUCz%rwAM&S*caffyOYOgvt|3Lmcb5CllQHh9bS zU27{Yz-5d0A)CpD@QHelSpa@P5fOork!PVZ!_30@FWm2aq*Qf zI#Eif-`$Om+$H9{%s>=yW7t#*btX$od13O-ybmS|3JG~!RV4^8D?%$z`vr`4A@2hr z5vRqMLk(FkU*1{h%WCiJ#3(2zq_lPRv^{ow>9}wM(@(sevM%DXk_@vq0*f^2{J(nc z`q*q4&uZgw1B&sg)o}VC8xM~fir8VUOUv>Upbb*E#R8jJkDWrGK2#GE6Wc527#O(W z2m3p#I+op^K2cnHrht)^U97j2!Xr6}vj{2PjrH|+19>|3%Oe%~Lp);pKgtl22K^RSKKk`22M5RWj~`)`)}u2&zLX6t z!`$u05nCD1F=~loxnXyFbm+bfUotW>C%q5WZ-j>dIzu=Sy#Liu1J-pI(9SDk_|`rj zE+8=Fit~@*yK5$rB8qcavIq!(ueHt1ys|4DA3sX3PSg(&oJL*Y<7?jRmWYJjYEjqv z<%?eCr@=u=fD79zNq;)Ir7~aH@qT_%;r=UbX$Hc+(@1a5XgjF6@*u3?4<6T_)D7vG z(|u}#pzX5vE!(d?H8-EgR!Nfu;90;Y4VXj7W#zm-F>U-%SE?AHprD}e%%6svQ&v>C zk2|kJP4ltDe8rUkpj;dQ0RgoBLPw(4*=m`xNV`*vINc4kUWFy)+~sz8#;O z9QODzHW8gj|CAmUG-3obNU5tw56;<+RjVDA!N(rVl`^2lYF&!Dh*}?VCW<29x~xA- zk;)Umqe5G{dq{BbJ6Opp&{58dds3e`b;iGaQ?6Rm^78T$Q&0qig%NEn4V~52)^3N@1|*KWJRuJ5SK^)Z(7igTL=f?4*RcJk(9yBa2NfZEe<>nHm48sw&l7O{6^n zM+@nQVawa*;N&Fl?d@%X4NHJCbcRkO2$*~bkm+!1sgE~r1oU{1;(G0grw;u`Y`s)r zd*Q+bW?o)(At50=`Dn*Ck6trHoLmS)#@Of9QF-DheEB(AWCLs_>Vn}WIq3X?5^0cr z#c6LZM#BCG8l-@tfq@D};m$NbHw>B#eh6Q@cj4Plq*Oy>QKbJY1I$aDe}5;^X?Jgj z_rFBeb+po214czoaWS!=?F7ctv#w7qhol^^%WPYujtFklyW+JB)^C83g&e*U8G0-W z018pc&(FVe_bwKcP`x3*Au)rRhbM2papY0ueI0pL;^1ijmB0hex||NT=>V#~tuKtPQIr9OU6D zTLu&(aJrq<$)R5r|5``Z+|p7~0BS8TQL#|)!!~H3PTpewT+S$Y1!d*d3d^C_RD+X^ zJ`8{csy%;|Di_VoxHGF`BUcr!vYl*&7OJYQo-t^sruGOx0efsnP>v09mtQ$`i_`Z= zXJ=vdBACj^`OiHp43KE+sIYKhFwj2+3XCv(% zKICnu#=^n^Z9>M>6n%z4JPhjut{pJAju3e9%Zp4nW?jj_pR-kFpD=h|(e2gx7R;9V z7FZjiLxBEQzlWQT!8}BcQ%EB3Jpy`h&!DegO%swn^5NUs*hBz0Yo45B0N5UNPl=h? z7fvAiJj3zsDWt?GIJB=vqxIV2(D>`?>lZ<8gRQ(rmlTx9UAWH7EOTD?Ar2@%01H() z{m}Bd_3i?OUfWuZs}pp<5VSZ6Nd(R&!wbKR*SeftZP@1;EHXL^>n2kMhy=s{lO})s zi`?8g{P03AC?q*&*u|EY@-aOnCO5J&QM=vCtbu14BEEf$2WSwKC(BqT8*f!gXz>B(Qeekt5w%3uwH2k5W>KDGNbM#a=f$%qQBcd%^h(+nC3OiRp7 zYAA;2o8I2y8=ITcb932_#*hTO|lBhU4dtjftbj$yR&(f&C( z7d?M{i~oI*5ac326|re(Xz=jy)kG8uCR}GvTTRsIEl^|nF^D9pcV5E#JUn;&^2*jW za7hj!QBf9dZWTDP{W!m0@G?6a)DXD^Y7v@iu2(=70MLiINsQcH3)D+bm90^h$-F55 z+hLe6eLO)q_{dSm)zQ-n29!EHGIEJdhD}*n8I%!q5nK`hN#ra}aTzs)Wv99;EKs}K zJ33tO9k{`I3U{aGo|&7izGA>u&bta)}xgIHQVZNhmY#(>;K&HP^I-PAn?-g zI33;HZ80k1xqm)OY`b3rCSh}=#8HkTdCIr1U*C?8-$wS}J#uXQmin>?fh_pBOt_sI zD!i>B5}}MJin$tWf4f(BJcNnCV)z0;yhN9+e3t&2FIpa?ARVL{c#vhi14OOFYu_2P zmFzu@@aNB;6J)<0MI_yVTY|qo?0uC3#*`S21GVr&^vYOG=AidtKYIY_f(>*U>MM7A zfA8Q-7*%c*8Xpj@Qzgm3JyB71WjHo0Fy-;Ny$74byU>Ng%O2yA@AuHZ)zK6*X z5b2v`sAO3$%iuJhBN{RHt?Qwbmt&!@Pv6NW`#m$#0l@l0Tgv&wNV6$Yk4pkDzcBcSo^+qVQ^hgZxd5<6oKP)AK5kRXTz0prn`SD`nm#n;rc_mrWq!^6`b=hR1&Mkq*1Xnp(23TBaBkqPGJE$ICqxD}grkjk|Be1Cp_ z?Qmyuwkd$n&cQ(gEq$A4+twHkFSUr{MM7>1UZE&e6)7f^Kd?namfW54JBtAoetxv7 z2*nu3smGk^^;2*C=csMy%a`c&mD*JTlyOhGq{XP~?%E5`s;RDCy=uJiwZr&uZ+j*) zQk-;ow2JD?nKLsxlRmcnJjc7UXQx+IxPWnCl$63i(j{H3-$_L@Nzl<|KIbpq-8 z+WLDkgr|{#PDMpX#YGNPhwUpTC)XpyMaHT`44ildwBZW+M&LRiQe8v^rKdBNJp85* z6&s5->r7xu_1;^+rOTX}ni3fe8vqI>1C+UsCrn#jKDfA|LQ5oYxY4JvkHmE614mKN z@vpRFIR3qxo0}RUk0)Mx-+(cq3;T;Gtf4?HLO={7Wb$wC?nVf2)5L@!XtO~>Mn1R6 zDJXKSGeLZL-`Yw5>N0p$?B~y)$LQ)NfXKwt?ka_+_TO^NLi!eg}_mm&Mjh-sx_Zxf-f zkloaEr(5uJ=Agv=!!_}NJ;e3}5MkkT3jpF%wES$mtBp+|;9@|rrD|=S$A>#7Q6+bq zaVxw*=Fm4X5|{vatbYX9FWTVtZ5hzn2SyYX6fgiJ_z9G(feBh(%LU3cu(#k57Pf{N zpdl<8p;4S3IA5~X7oJOLc-l>adk6iPIGiu42jn&1GJ&tcx1xUtXZ@tx!16f_?ssO`?n@waz>8L_@jyydsPIS=X)d(Ca1mRptTHaZonD< z@9y@H5n{T8x4n>lqN=LelVkGz+qdbOxl}*|7KfdkoojFcXQ9A{hMhsH6Bt3=0c2<1 zn{g5ptkQ^^z=lb zpu|;|M!^nZ_*}k#Kx*^VdYji zSdXI`-`;OE$K?6PYJb$i((v4(L;@Ba9UTN68hLF~Kwo&Q7|(SVR7Pl&vMDXG;)@Pp zK|yj7Pz-5*`0!HYr%wJkIa(3>sQCCNpmPifMcE;C*Ro}j9HJJ3lOv73aN#agJ8CNl zgF51i0>v)-Gv;waTL<;?t!4Ds_&ChxFz^=iT0tjn1~HD&UsY5ODiXni^%7YE-UqHf z#%jnWeU81hhjQlk&b<2@n`X-!gEem_=l{vJLA&v4f}E8EwM>Tp@Fp|r=6&XO{2!WW zfyN+qlSTXz{m`&W8~tB4RRpY2#ZLg1|1hfDt+|DSsJptlpzX)I(6ggZRLw{h;B>>m zhT_)B_&MljBK6MwIcgU{QgXlL+jeZ<;RTDLB&n`WihG*YzrSB|tkK7)mou_avT&hy zm_AL;)xe9Dv^A0;5=@S%{nf^{z3nBedGTOWjtaV8Hh9()kM*`IsMn{9>NW@fo?QWB z2#rSHX%0LM^FZ9tec?i9s#vXBTath!B`g&QFlvw+??5w7;D11hxWLoXkMzpS7Aopr zFU=x8Fh~JBHe*65M(4=L`~WDJKv_@&T0SZk=-9<-D_}? zpaki_8}khfB|JDdxB{(OmacnZg8tN*^KTD+ZI(Rv%<2g)C^8knV1@>WxYYGk!f=ns z1clZup_+hz z5!waqFM;Ee-7D2{L@G7(52=T_)7jY^1~> z$NyG3o`uIo#R@A2rt3~?eXjTp7QeZ5WJH9lqhq;jZ9u?DSffWYd}i&uX27G0p_%6! z*1wsU73+dLKs5U9`yi45LPn>or3S$*Y5~XFD4Gc%7;Ev32Pa`_!ZZ*FJbf+@u^tu& z@&+py(UD-uB9ahja&5qlo<*4SgItljP39DsZq4%({;9-ySwH`FE#=C}3N^nuF=D^1 z7qExR2Me}`i$$qU%HyQ}x`~#1?4w0Naqe8`NkYo0-St_Za9Sb}9iyf92)&OFT~b0i zU%djM8^QMaNo8ex;7cKD=L@)ewa>A5s^^A0C`&3YVn2O`&Pq&6O8^dx-w~EA0YWI_ zjlA{ZNZphGvU2 zz0%qUgnJI{{I`HFdO_$SqU8_bOCSIV@VDXz&zf#K*$`qT?74fTVRt4}pNe<%%NL|& zxpy}S20|n6xH>Cvw+W{pARZWqY&dNI$dDNT%SugooZ#6JArj7bkt2C#TazLa7%ILs zHNtya3mUUhk{U6fOK~^s!rcPlLzW3h+`V&Upl={21oVvxac+51pfIl`mQ60gcoGn2 z^S=(!0EEs0X$rrUp}L7=J)pRUTsKXD9EQ=0=|^QK+DO2W-tGo4hB%l!7X25&Okz*? zP|n{Xk0Wg|o%QsU&=QhG(7T^luC}OdqOVVTv^UgfQSUzlXYc^RHLf$kY($CEp9Ci{ z;O5JE4AkyGU4dzJMn(J=I<(MCDN7wW8?BoIBP04t*&h=V8GH^%K*mJWT<`)($jNb_ zRcDzzPktp7e8(J!2kkq^RPYTsk3JM^_!1Co=m57XCyq1v%473sY>9)`4=%@)~ zLa4>u%SW(1vQ~94t&$6y1Jq!?Bh?H&QQ-#9gPZoG?|Ojh2}BE64yMBqA7}|j%R?{e z-<{x>-aXo1RZ~^<1+)d|hZKa+M->%dT3+~^_4Stp(caE#kt2@9t6h@Q8L}_{sG!(N z_Z|j-oHQvmcM3;FG_2reo8e*YLg;Fx0aCueS2gUev(rBj9m zsDt2>32m=y9mR-7V`fujL0O-SREU1LcnUDEnl$w|ZPD160eBxsLq(1ZU-vaX2>RS@ z38B#6H>$S70vl2WPyqtWq5BYqQgOG=_hv4oFo7#CE-pSGp(6$e=qNuY2ak+R**`Cj z1K9NR;$n!)q!*Q-%~;QT-V*xyZ8FrJ(l!3?)%|v|E(}GJ-utI-M4Xp|se*-q6Y&I^ z1Go{sAn{r*ACJ3CPKOIlpiqcC0u8-M$H*&SXTswwSc`+A&EU|kgHgu}`U=a92&>^U z@FX!#`#-IbnN7uQh;ek}2MlHh%odrfp!W;8Z(Y={w94Ia*7#SzYH+u;(P9Qkm&~?? zOT!$ZROs5koj{shWu zwDQxPS@mY>+tU|zDonk*$sq%w11vSwV?{W;@%BGLNb3MT&RJP0UrS)ktnJS!0)s!i z5~M3E6i{!?@mK1(h)&wO0~1BR>SK$4zb5D;O=DwUj~9~8H*{ots=vGg5dJfFpmVXt5lrf~}eRfOKjdA7%ZTm7YG^dget* zTNW-U0T$z9Kffg_GO3`csXH$&TWB4FsG+y@N9nl#t2GfL>q>o+QHaGK<7xP!%^( z^L*<)3A5^c%hCQ<#-BMwqnQ9x18Y$gA)pC{F)#(Bsx6F$kNo3P@F19^0~tj{ZUL0Bk{N_Yym_DgkML&B&iOl3lOduIrZri2rZK z%m1Nor{hgEVvIp^2B*OooZ66k7cXCK0d2Q8PlvmOmw4gBaMw_DG-(zKF-hmf1*nMs7muZhm0J+)V{kfx!gT*(s|x#QaqyT~hh09acEZkHjD|rz znwXR{!;z>UY&oT>5&}rnv7o>@#SLsKVu)_ROhqURjG=1pLlNx){Q&r!8G!2Z{kbs{ZNBa4I8jl* zTU13Q-G9tGSdO)V^~dS{^9MVGU1R_tv`ilT6a;IlS)}c+uC~4_L4wj}aH_GJehkd9 z7$_PA*jWC%OGd|+z=$r7NJ>lNex;mp$LkqsoI zJ3OolYs`d%Rs8(?Aaq3r*1MA#>v;Wtj06`h{nFO)7^p-Af$^eT;03q{R3HeOEehCJ zn^^Ffv=TvPY_j2Yfmde&kABNyhtwCl=q>7 z0|_N%AgB%bx+U1%*EWd&1^R<&WWH=9K!*4i`35zlFyWfOZ{%^`ns*ORi;l)-U`Sg` z%cHNzO~eQC!_9g#(Z9M{NK;d@UzltS(i$cpawAp}_K6c9SNe;4?WMnbiOIgE#fODd z!rMnDAgPMvFK6MeNJI}L{Y57txzB$oJumGI!)f?%BO)S(-#Fz-wypNOIDxn`6_sGO zVWR*@n)Q_Az0Asb2Mh@k3fh*D!NI*FMo1!=>pv!v`EB6zfzv*_D=KuX zIUv53Sr1SaMGUdPZCahl%`r%1TSkCTZcc{`zV|IH!E>n|%H`(0QYM4WU>b;gE2*q3 zwEs@!3t5ZLZVW-DrJMttlDFT>(Zr5{btzUpsLPQ`=sLMG(AVnULb#?^_X-POcXNYw+*0E zK&c|&Nmox#67SL5x7RhbwHM1%;D&ei_GUoo5U?}OXa(a)DX^@pOg5>Y;4%b)Dt`VB zX^)n5ixZ+sZ(Gzt!sOuaP++9u-if2lC#K+>$|)<$f}DrkL~5hk<}2k{vwQbmz?24X z_<67HIgJhtKjj~gqcRFf1|-sgDD_K4>bd<~G&(hqpBr2r+752|7bNo?X?p0@I`B;+ zEOKeMgvEP*Sumv(>n)IeWDCO;UkDu5=Celv7YzvwB_SqG*D++iuwKReoXG2#nm<;Z z4~y@9*L!%ba3D>0fEplSf9KVS(N{-x~QGYAyZD6CdAkJ#M>Q?@o~>1VIr9sRo6Gy@!EZY%?wb5QiBMgvtK+8>i;h@N+!(K9EDnGJuM*yEZ-0 z4c;6Lq$k?I^;+-b>z)F~`7JP!Cy?w$Y%%1{Usam?ma#x}fjHg6?PWA%T~Ow!*$Bvl z4OUephD<#u){g+c^y|(K#NLPB?LcycAZJ0y1+jvlvY5EJ&w{c9(Pb%PW8?6eVDcC9 zq0-9|A)GK1O(1|z_!)PFf zAxrDUap}_gu^NW}APHa#WB{r{(v1SCAO9`t!f#{0v1_dXfkANWN%_n=iD5wI>{M5T z>%~AU6^U{IYxD=g2~au#(wou00ph^k1W25)5i$^NFoa+PAr%!B2^ks75g0)Qb8gni zr222ejPFDo$U3a32)TJh!nIrxUkG%%AfTn80NaBh0axIIgXH^Acs@QpQ3(kbfg%Uw z<>i4;-aqUNeh-SVHG+-U@z-CJz+YnE)z7#BC&D0NLm~uUe~P>M8g|4d9UT z5GqBI^5y0Hh}#Z4yw+tk?y#8$BYPgvj7mmb8!rHNeJ5$BCL$ z;Ola%pxeNg>F)sN|J&ahpSe@n$Ammh;-j(SQ_ZH`lwEXN%kay7D0vxWOrg}B$Nvk& CkAABF literal 0 HcmV?d00001 diff --git a/doc/lstm.txt b/doc/lstm.txt index 140e5c62..f9896eca 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -6,9 +6,172 @@ LSTM Network for Sentiment Analysis Summary +++++++ -This tutorial aims to provide an example of how a Recurrent Neural Network (RNN) using the Long Short Term Memory (LSTM) architecture can be implemented using Theano. In this tutorial, this model is used to perform sentiment analysis on movie reviews from the `Large Movie Review Dataset `_, sometimes known as the IMDB dataset. +This tutorial aims to provide an example of how a Recurrent Neural Network +(RNN) using the Long Short Term Memory (LSTM) architecture can be implemented +using Theano. In this tutorial, this model is used to perform sentiment +analysis on movie reviews from the `Large Movie Review Dataset +`_, sometimes known as the +IMDB dataset. + +In this task, given a movie review, the model attempts to predict whether it +is positive or negative. This is a binary classification task. + +Data +++++ + +As previously mentionned, the provided scripts are used to train a LSTM +recurrent neural on the Large Movie Review Dataset dataset. + +While the dataset is public, in this tutorial we provide a copy of the dataset +that has previously been preprocessed according to the needs of this LSTM +implementation. You can download this preprocessed version of the dataset +using the script `download.sh +`_ +and uncompress it. + +Model ++++++ + +LSTM +==== + +In a *traditional* recurrent neural network, during the gradient +back-propagation phase, the gradient signal can end up being multiplied a +large number of times (as many as the number of timesteps) by the weight +matrix associated with the connections between the neurons of the recurrent +hidden layer. This means that, the magnitude of weights in the transition +matrix can have a strong impact on the learning process. + +If the weights in this matrix are small, it can lead to a situation called +*vanishing gradients* where the gradient signal gets so small that learning +either becomes very slow or stops working altogether. It can also make more +difficult the task of learning long-term dependencies in the data. +Conversely, if the weights in this matrix are large, it can lead to a +situation where the gradient signal is so large that it can cause learning to +diverge. This is often referred to as *exploding gradients*. + +These issues are the main motivation behind the LSTM model which introduces a +new structure called a *memory cell* (see Figure 1 below). A memory cell is +composed of four main elements: an input gate, a neuron with a self-recurrent +connection (a connection to itself), a forget gate and an output gate. The +self-recurrent connection has a weight of 1.0 and ensures that, barring any +outside interference, the state of a memory cell can remain constant from one +timestep to another. The gates serve to modulate the interactions between the +memory cell and itself and its environment. The input gate can allow incoming +signal to alter the state of the memory cell or block it. On the other hand, +the output gate can allow the state of the memory cell to have an effect on +other neurons or prevent it. Finally, the forget gate can modulate the memory +cell’s self-recurrent connection, allowing the cell to remember or forget its +previous state, as needed. + +.. figure:: images/lstm_memorycell.png + :align: center + + **Figure 1** : Illustration of an LSTM memory cell. + +The equations below describe how a layer of memory cells is updated at every +timestep :math:`t`. In these equations : + +* :math:`x_t` is the input to the memory cell layer at time :math:`t` +* :math:`W_i`, :math:`W_f`, :math:`W_c`, :math:`W_o`, :math:`U_i`, + :math:`U_f`, :math:`U_c`, :math:`U_o` and :math:`V_o` are weight + matrices +* :math:`b_i`, :math:`b_f`, :math:`b_c` and :math:`b_o` are bias vectors + + +First, we compute the values for :math:`i_t`, the input gate, and +:math:`\widetilde{C_t}` the candidate value for the states of the memory +cells at time :math:`t` : + +.. math:: + :label: 1 + + i_t = \sigma(W_i x_t + U_i h_{t-1} + b_i) + +.. math:: + :label: 2 + + \widetilde{C_t} = tanh(W_c x_t + U_c h_{t-1} + b_c) + +Second, we compute the value for :math:`f_t`, the activation of the memory +cells' forget gates at time :math:`t` : + +.. math:: + :label: 3 + + f_t = \sigma(W_f x_t + U_f h_{t-1} + b_f) + +Given the value of the input gate activation :math:`i_t`, the forget gate +activation :math:`f_t` and the candidate state value :math:`\widetilde{C_t}`, +we can compute :math:`C_t` the memory cells' new state at time :math:`t` : + +.. math:: + :label: 4 + + C_t = i_t * \widetilde{C_t} + f_t * C_{t-1} + +With the new state of the memory cells, we can compute the value of their +output gates and, subsequently, their outputs : + +.. math:: + :label: 5 + + o_t = \sigma(W_o x_t + U_o h_{t-1} + V_o C_t + b_1) + +.. math:: + :label: 6 + + h_t = o_t * tanh(C_t) + +Our model +--------- + +The model we used in this tutorial is a variation of the standard LSTM model. +In this variant, the activation of a cell’s output gate does not depend on the +memory cell’s state :math:`C_t`. This allows us to perform part of the +computation more efficiently (see the implementation note, below, for +details). This means that, in the variant we have implemented, there is no +matrix :math:`V_o` and equation :eq:`5` is replaced by equation :eq:`5-alt` : + +.. math:: + :label: 5-alt + + o_t = \sigma(W_o x_t + U_o h_{t-1} + b_1) + +Our model is composed of a single LSTM layer followed by an average pooling +and a logistic regression layer as illustrated in Figure 2 below. Thus, from +an input sequence :math:`x_0, x_1, x_2, ..., x_n`, the memory cells in the +LSTM layer will produce a representation sequence :math:`h_0, h_1, h_2, ..., +h_n`. This representation sequence is then averaged over all timesteps +resulting in representation h. Finally, this representation is fed to a +logistic regression layer whose target is the class label associated with the +input sequence. + +.. figure:: images/lstm.png + :align: center + + **Figure 2** : Illustration of the model used in this tutorial. It is + composed of a single LSTM layer followed by mean pooling over time and + logistic regression. + +**Implementation note** : In the code included this tutorial, the equations +:eq:`1`, :eq:`2`, :eq:`3` and :eq:`5-alt` are performed in parallel to make +the computation more efficient. This is possible because none of these +equations rely on a result produced by the other ones. It is achieved by +concatenating the four matrices :math:`W_*` into a single weight matrix +:math:`W` and performing the same concatenation on the weight matrices +:math:`U_*` to produce the matrix :math:`U` and the bias vectors :math:`b_*` +to produce the vector :math:`b`. Then, the pre-nonlinearity activations can +be computed with : + +.. math:: + + z = \sigma(W x_t + U h_{t-1} + b) + +The result is then sliced to obtain the pre-nonlinearity activations for +:math:`i`, :math:`f`, :math:`\widetilde{C_t}`, and :math:`o` and the +non-linearities are then applied independently for each. -In this task, given a movie review, the model attempts to predict whether it is positive or negative. This is a binary classification task. Code - Citations - Contact ++++++++++++++++++++++++++ @@ -22,23 +185,23 @@ The LSTM implementation can be found in the two following files : * `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. -Data -==== +After downloading both the scripts, downloading and uncompressing the data and +putting all those files in the same folder, the user can run the code by +calling: -As previously mentionned, the provided scripts are used to train a LSTM -recurrent neural on the Large Movie Review Dataset dataset. +.. code-block:: bash + + THEANO_FLAGS="floatX=float32" python train_lstm.py -While the dataset is public, in this tutorial we provide a copy of the dataset -that has previously been preprocessed according to the needs of this LSTM -implementation. You can download this preprocessed version of the dataset -using the script `download.sh `_ and uncompress it. Papers ====== If you use this tutorial, please cite the following papers: -* `[pdf] `_ HOCHREITER, Sepp et SCHMIDHUBER, Jürgen. Long short-term memory. Neural computation, 1997, vol. 9, no 8, p. 1735-1780. 1997. +* `[pdf] `_ Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. + +* `[pdf] `_ Graves, Alex. Supervised sequence labelling with recurrent neural networks. Vol. 385. Springer, 2012. * `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. @@ -52,14 +215,15 @@ Contact Please email `Kyunghyun Cho `_ for any problem report or feedback. We will be glad to hear from you. -Running the Code -++++++++++++++++ +References +========== -After downloading both the scripts, downloading and uncompressing the data and -putting all those files in the same folder, the user can run the code by -calling: +* Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. -.. code-block:: bash +* Graves, A. (2012). Supervised sequence labelling with recurrent neural networks (Vol. 385). Springer. - THEANO_FLAGS="floatX=float32" python train_lstm.py +* Hochreiter, S., Bengio, Y., Frasconi, P., & Schmidhuber, J. (2001). Gradient flow in recurrent nets: the difficulty of learning long-term dependencies. + +* Bengio, Y., Simard, P., & Frasconi, P. (1994). Learning long-term dependencies with gradient descent is difficult. Neural Networks, IEEE Transactions on, 5(2), 157-166. +* Maas, A. L., Daly, R. E., Pham, P. T., Huang, D., Ng, A. Y., & Potts, C. (2011, June). Learning word vectors for sentiment analysis. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies-Volume 1 (pp. 142-150). Association for Computational Linguistics. From 0c26a30f81b119b5338c7b886c7b72208f8cc8af Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 11:00:54 -0500 Subject: [PATCH 139/417] Fixed section markers --- doc/lstm.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index f9896eca..3380875e 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -124,7 +124,7 @@ output gates and, subsequently, their outputs : h_t = o_t * tanh(C_t) Our model ---------- +========= The model we used in this tutorial is a variation of the standard LSTM model. In this variant, the activation of a cell’s output gate does not depend on the @@ -216,7 +216,7 @@ Please email `Kyunghyun Cho `_ for any problem report or feedback. We will be glad to hear from you. References -========== +++++++++++ * Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. From 35b3f9f1af3f0261cf104f6535ed7255e9ede8da Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 11:02:04 -0500 Subject: [PATCH 140/417] Update LSTM tutorial description in the index --- doc/index.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/index.txt b/doc/index.txt index a87f66e3..7c6605bf 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -53,7 +53,7 @@ Recurrent neural networks with word embeddings and context window: * :ref:`Semantic Parsing of Speech using Recurrent Net ` LSTM network for sentiment analysis: - * :ref:`LSTM network ` - Only the code for now + * :ref:`LSTM network ` Energy-based recurrent neural network (RNN-RBM): * :ref:`Modeling and generating sequences of polyphonic music ` From 54fbc9e315d31515053a0f0a046c042d23765175 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 11:05:47 -0500 Subject: [PATCH 141/417] Add the LSTM tutorial to the high-level table of contents --- doc/contents.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/contents.txt b/doc/contents.txt index cd7281ba..3b7a16eb 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -20,6 +20,7 @@ Contents DBN hmc rnnslu + lstm rnnrbm utilities references From 63d5161ccd85f8ec1f0c04d90470162f0ac4b0ee Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 11:06:21 -0500 Subject: [PATCH 142/417] Fixed typo in title of LSTM tutorial --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 3380875e..f78fb5ea 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -1,6 +1,6 @@ .. _lstm: -LSTM Network for Sentiment Analysis +LSTM Networks for Sentiment Analysis ********************************************** Summary From 2e022a636c62ed7b8f6f536124a22d791e4a179b Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 12 Jan 2015 11:18:46 -0500 Subject: [PATCH 143/417] small fixes and doc --- code/lstm.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index c7f0c85f..d23c6e76 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -19,6 +19,9 @@ def get_minibatches_idx(n, nb_batches, shuffle=False): + """ + Used to shuffle the dataset at each iteration. + """ idx_list = numpy.arange(n, dtype="int32") @@ -381,8 +384,8 @@ def test_lstm( # Parameter for extra option noise_std=0., - use_dropout=False, # if False slightly faster, but worst test error - # TODO: This frequently need a bigger model. + use_dropout=True, # if False slightly faster, but worst test error + # This frequently need a bigger model. ): # Model options @@ -502,6 +505,10 @@ def test_lstm( best_p = unzip(tparams) bad_counter = 0 + + print ('Train ', train_err, 'Valid ', valid_err, + 'Test ', test_err) + if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): @@ -511,9 +518,6 @@ def test_lstm( estop = True break - print ('Train ', train_err, 'Valid ', valid_err, - 'Test ', test_err) - print 'Seen %d samples' % n_samples if estop: @@ -537,8 +541,8 @@ def test_lstm( history_errs=history_errs, **params) print 'The code run for %d epochs, with %f sec/epochs' % ( - (eidx + 1), 1. * (eidx + 1) / (end_time - start_time)) - print >> sys.stderr, ('Training took %.1fs minutes' % + (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) + print >> sys.stderr, ('Training took %.1fs' % (end_time - start_time)) return train_err, valid_err, test_err @@ -547,6 +551,7 @@ def test_lstm( # We must have floatX=float32 for this tutorial to work correctly. theano.config.floatX = "float32" + # The next line is the new Theano default. This is a speed up. theano.config.scan.allow_gc = False # See function train for all possible parameter and there definition. From 4e1ec30451a0b2c9bb199e76c2fbc640352d0632 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 11:51:46 -0500 Subject: [PATCH 144/417] Update LSTM image with labels on the arrows --- doc/images/lstm.png | Bin 13104 -> 13780 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/doc/images/lstm.png b/doc/images/lstm.png index 5620e3a0b3fd7824eaf281a7f051e50640fbcacc..bf64ce02309106f72d8bf6c6db96dde50aa34ff4 100644 GIT binary patch literal 13780 zcmeIZcQn?2+&_L9*=2{49T}M^*-2&+S&@~7kyVuJtZYTnvO-o?k+Mh8P(}$Uk|adP z%KSdwpU-{1=RUvt{{Hhj_xYXgAHP$5KIdGn>wUe(^Z9&?=Tp?tBiftkw$YJDq|LfI zn#Lp&Sp)us(oo@*nz4p5{EzCSzP2W5gZPtCo0~);@sV^j519DeoBVRw!sPUZ+Vmpn zQ8j-=$gM=-X!4NlbdJ#`A=x`68KbS-o*uP46KJgy;43@xSJ+I_X|^HjxaFN&`jR6o zccLTj?~2sBlbv`&Pl9f7F(I(ckkaeg?fr5W>TP>QyvG9G-CL2ndiS#54~a@W8h&+6 zO(70mxg2pm0RdxjiffUP+0RKr0s^8kq^L+8JqC3Ffkc-7YhF$a@7i?D(`T}xr__7G zEawehL51gVi~Y#!WWAIFRKa9A^4~^Bsp;vrY?oCH4-Z%H{1(=A`Wo4g2v_&Zk408P zKY!+VacRomyng-4!eqa=H5C_A8pXf6eRV3&L@X>KV%zH8G561v*?Xw?oZ`v?GhBO1 z$z)_?@O(|*mJ7kBWa;SX?cLmLhI@#g3 zQfvdcHgDUbzP3D1MmlupkVUB-8Qx=m`gFKf+%|q?W$xq0k5_zB`!s$mDWKq$V)tbK zo!Erg9$vLT?kiWWlvY$6djG&c#(hAUhJVlU@oS%#*msS+O%Ga+;!*Ma`Z`(smV&21 zO>OPguGa&(A`g-ai<6Hk#I)rsCI8VyhOn z;0ZCK7l#;n?;iKBzQ@#;FJE^2kjKoz%39_%W_(Z4n=UFU%JB1# zwUtHYZQG)M*KhD(H_drQS#~ZiF|n}>=lUKU8vgX?(IX~q?%2pk8U`L^&YoiHgSxu3 zA)%qy6B9!aH;r%J&_8?jY-wpYp|i+p-#rCS^4(TX{Kwah^{dK%$-VRNSTZ(#zOS#3 zbB`laOZ<+QmoIfzN7XhOo5JZk^Na*FwY6PV|M2P33T&=f9xrTaYdie8%DeEBH!1Sk zwQJ5P?P)4jB&LavMTH&feP`2YNhjak+duxKa==%g#MJ-9wXk&RmW)HeK$vOV;C7 z^vculyLj60g0uV24D{^ZUSuj`y>*qIfR_iO8psq$RBBPHhfCf|C?9LAqiU-})r zJU>A%B~_xghR{56>-Oy^T2^7@vb~8K+K6|}?PR@AT&+5@b(8cC7+G41-Me=$?CRC4 zRYmyLrta>FuD!;`j>&InIP>;S6k^46DfpqD0eQ!pH~I>SiqVOQiLGvbJV}vTcUzh` zFV4&`+t}EAL*6B?Fa7-+^}yh6K~K@xdZ$*phT{`M`7gf{J|0h+F)PqvkErUv)Aafs`=1oG zti5>gBJ=4}PU|YKD}{yP5e!@*@-8304%GS?uJOpq$qkK-9iNEVmlm}CSI?D;i%Ume z|KM;-ocP-Q^WSOQK0nR0S#(-3bX}p`)1PZtJ?J8<`g*^TlDYHa$Bz%%+YdRc2$1`1 zxpug>;R=Q4{J80_&(Bz2y?UkKGtF1GJTBgLcORqL>h!^6vink|n;efH=kgB-m}#Wu zu=hv}-sYUJP#5`fn;Ou=!S%phe++j@Ys!w0Petj)1ZOguW`@F4Uw(j;AY6{ou zfBEudo>4k-v!T?!S^8YBG^yctN$hp;IDowfP%5WAQOWY>g#)_X|9!ze|r&Q zQSNAz{6^R?Efx`E|M{t>!sS^ZWfc|A@eiU8PdsCB{Z!uAc3ZA{0*S#o(NofjjWg}o zQB>EaCMU-ld`8>#?H&2%H&XQsYHDhG+&*)n5=15>T-D3bICU!9xAsV?;t+oNe7k$i z71e-c^&DgSpBG5Bx zFF9aQy|sS(d>5xSi;Ih&sPpGSo)G?&Z%c@paLK zmo)JUW{#O=B&;Oey(^9?Eh>EE-mZJ8slc3Y4^K~KBO@bIw|#PQ$HKzGXsoQPjOqge zWPGOWt|ui4k|L+4r^Qx+mI|6K4F;HdO@1=n+Uy&>SN41Of{yj z)6Sh5k!u=!>Ups9FX{dJ_nnyssoQSJP?VLGMR(ihJq;=_SLNk98ElmQT(_5qHxVO0A=(*Yqw`iJ6Tp?CV#% zoHxg~7a5HvrSXjUsW016$8`=LejZBA*im!&Le0ul^)01KlD{{|@g`Rpyb1SD|M<*X z<0^*G7i$Mil$5D;G&PM+l319p-{1@ed)b*rk-d!z1zJKv;oQq5j|duHHnmrIT3dD~ zW_!_UYFajmG+xuv)7LjLGkf60WsLP`96ic(?%cT>k}rgA-@V)17JD{`Mio?t0=Tw4 zmU}&< z#kH5}H2l-)8>`d}gO4*(uJSXFQ4ue4s|D6Hg;gcIi#?+G=>PvOj!K+(PG>5x0G!>gjLdLzQ zhXi;c+$m5<52i?3OFOhM+*1wM#vyqkI5l+#K9uWqTT}QdW3#YvuS-cSRkEJx$Ro2N zOZykm>|a^wqskMsgEi?2C0CuRwlH@y{C`Ia%QbV1`SaKcy#m;%P8Q3>Scrio~!(k&UQ4Ro)XsIr8w`=O>dFIT65L0F&Q{0*5_g}wGNayHzGn<4Sys!y@-I3jW{4xc}d9|5KS>WSR-D+uEL&)UOwKaYaA#p4i%j zno??iFh^g82Bg=qyC50NoYt6@KdkEzlghc*H$*MUPKM*PE4New6mLo#zwgC$%GckA zQim$vrq;I?yb3>TvI+cqI!B+6wS^=UddlAZ3fQBVHI-;vTWpHxwJ8rT=6}nANfZ#m z(eWp1e`G8d#kJT(y_^vIt*pFMG2!FqSJ)#c;LFxdme~2|$rCLkFFT3+6Cx;MbW zYd3FFxwyEnv9pVsWIBmTNQ8kTcAaK&+MGao<+zEGGQ%iKYiadQyZG)E{1>7kvY|nJ zlbDYsO#q?+Zv+l#-^V zrb-_@(#X<^W8&e7fB5hr-NYB9==)s5t0g5;1ZmdS*DrVMBu^4HXezR*6mzXynFB9Ki(cRaI4%s_zv|JG?E&xD<~;Be0+F(m#Aod=@dT4#Lm9S+}xZ{ z58d7DNg{?VV9hp8)$+#|J-t;X{ocKMr-j1(YjSev>(^jV=Fc^klSa4d$}p(QIyxI8A}HSuvX7=>BM zT}Un0{wMs>(oFg(lBgr%1S79->1i4q3%>^wAUW(`CGOm{d-tLGz_qk0+xwQRnp9_I zzrSkg?3^i^ATjat^J~!X`}_NAdU%xj9v`d=2rnrqVbTDfXPTLo{!hHSY2nf1$Ke$f z6~JhdO7^W=qoFqR0z`y_XhA#_xZ;-UO`q9cA5(2t`-A3(9+byw5 zD_q)*P%OUA~-oVf5X2 zP=9}amP2d&%eJ;aN?yYeV0^he1l^ z^*cXuGjN(sOG-@qIzAq~zP4iSGx;I^SUQgW(>K$MQ*-Mx2|;!(v0MbHV^;{=ERZx)Td&&`lS_wQVVqbpPpE(l&$uGEn|M4$oN!~7cNsIm?uh}Un zDNQU(3Kn&+nScV&q~mLz`yOQI$@_C4B3YIys0GblhP(|rw^7t>a7vR<=D@`smfHCHD?=Auxk5o*fEz59ge~$7 z5d-E_&{60A_k8Mt{+t4$Ey)h2y8prtWzX8N_qh&>Q(rjuxp80oJrD=7(F~IU=ih>) zg#wFeS6yKs5hZSE$!Bh&htb~N9=G^n-=Ke}-{RER^5oaQqv@?canEnhY0I6v>8dYI zHvM>emzjrWOHfb{sR1&f%;~*w?DluhvS|)UY3wY}FmMm`xu@+Oio8oqWGOrKie+eW zayJ-ZSWZrk$At^5I6nbIO6`Dc(Av%cpW)4&HcZ5NdT778KvzrwkDg7S&0|!9Wy!0NqogPuwWg+~ zjz_2Uy~aP#GV&;^8ymA+ocbJC}Z}Ifl=UP9GmBrt!;w3fry+nm`Q9ghtz`rXf z4y&uHOb{U0-O}>1m^BaAJ}SPk9*_qnF0QD-z?Gs;-o)d}99qZA3O>=&(l#cjtw)B0 zkb|%2`YrrwjFGCd7#p`B^g8Fl*n4WIGCETot!OfxpJgZipa|>=)wg7}DG$l?t(j3R3?+v=`MQh)Dba>o=^_J+l}2;hfN;WQ!^4}A zq+}$N>fV~mav<9U9?5sackOBbClD)_A$b>`ZBHYXC01Smrvi5X!{u0>nCuw_^{#vZ zwONG^tAlD_ArZ|G8~y#;NgeRt;J-e!uHFisL1%6$r+-tT5i~)pJXJNoGSm6{OncD! zv9L zCD2YcN!T8MOJ`}XdNbfJKl7#$cs$m$D+-#t}98>+AxvRcQT-KW1c_M%KAzbUq< zjfO)u_DtZfgk?F+lP6C=X{)M=%FEdxrz2xxnsd_wU$4lH=_0$G+F?QvVE@pzo(Cta zY7?~Po#c1t%~12z|Dbr_Y~K)$c^*LoL=aR<)FP9cFna%twr4?pC*RxwZx&~%4cMO< z%T1sC$X2k$J(O>f*cu4K_4M=v6p2_~wSn(VjHJMo$$j_sU!c15d*B5?bf!(N{p(%# zx<*_qid8}TJ^aVD}2yps7to2(w2@>iv^=z~h@T$D{`2uPM69W5a ziH*wlA3yG+YoM2M(9j-LlvBlbk4?b5sl@R|VYaq!u5vUU?IzF~z~S(#w=6O)?gf|~ zXCULNckf!{+LfE9uQJ}a_TMI|8R1m^>RGh0j%Ns$OylzEK5n-gUWT;OO!a^(=s*YbBbLagtbAhZQLKjPA}Mj^acQYK z&K!g~mSFcyVYEvVCG|HXW4yx1cIBfOXTYZHed;Mluo9v#0;z%hvC+}dd8{pam%Dwj z95dF@20nPqe%D=vIry_HhaM4@k@}I_TxKhn#{!#;$>sot@nYK;$L z#=W+-Hn4jaEo`6EwVvVABW1)^hOm|0mviKu|m%RQ2Ma%rsLw%pl{x&S{k@+fAe z!>SQsFW$OP48ow3=p zT_PgdptsS0;b`boQS113Tutrbva-g&A~UIg>odBBh8*xz1rT*8YF4=cTur}2uTSg> zAQhO1dH(8^@$N?jKZO2MILx7tWEC4v&$3%;L3L>2F`+Rv8Q_C40#dF6@g(6 z5GE9{@87>?pl&JN_UN%tt!GG{Bg=c0)qv}i!DirGlTcKlJg)PItyuk~tk|LG6^s)1 zE=@&zp0dJl`SQq*AKz*rKHNx0;MdoYb5XO?{5YnA{vZ%#rvZz)(qc3 z9r**t___XXd2BcgU9q3vZEEGtzMCFZH!MO*DtLS))zs8{g_I%Mt)l@pH9pKB4@-0J z4LhHDI_z;D;MR`k?z~*z5<<2_2|dsR_4;*WB**Q~!a@s*Ma4i06M3O|NwM^S-Dl59 za|1FyiKTli63C1GMP>@^C4Ie3G@kJ!5_^Ya7qH7{Hv03EgT`Dl7jymj?O&k#!r6qX&RwvBJ!8jKmgq$Erk zQBl!d5)vkARwm|QOoUt0_Vz8&nMhq7{bHfJCUMm0?R%Cd=mM=**Vd{`4b~?>A$ijb zR9|8QZSB_Ed}-=)BWmJIx4Er7;l-%>E3~yOZ6p>r&i?622FWOQ?Z2tEz3Yze{5aec zGB8gKm@P#8sT>Tn8kTu5_B#{=bNSEfmKG|s?kdDThfP=hrQNpAjX^|21ctQb7wn9R zkuky2T?tP>S{^boy6zid8Q8E*IBi4FS~T6`66Enc5A*gVr@|}O1+`DEum0V&YnM7o z(Mbmf8RrjqxdS#@T2$!j#O>6-X$KgysBmT%$?Luk-|fbY8+jWxCMG5muCuGWW5%cs zWkDN(^I_EpN-zRZX_9;?Vr7q07bVmMdM!;InFnQ{v9`JAW@M0n2Q)M^4Nw{(NDz1k zKQm4~L){}F9-bDum{;QB42mjWk%X+pal@cUFO+`QWG#hN%N~syuhS7OT8500cKx(x z?q??jumALcWJ3?W<`Fn=0B7`4#jNRc2?GS3k0#g_*Kgg5gpL-sP86NBCyn=ls7s@z zf9mYnVNkC|_%#_`x+VvSW*!31$1P&c4R!;e0YyG{Z{`i$R3JFQ`r#t*155yP++m4J zG;mEgtZu zUSefoxgq)Llr*X0Cx|rxjKYU+1?T9eoP>#8ODm3} z@q_XC^Rl?PuSiWWVNM_!P@?NTViN?630=zwgjClEQ~o5pFCc6S(mxs(Z{l-8&?$_g zQ5m0bBUlG~ux51c%lsDIBGPHAJch_E%N-l11w)SkKBVebLx>p$yaD{ni!)f>STaAZ;@^ayy!v=3$8mBFBm9q8V}hiNW%dQXV6h;$(z zG6t>~pwc&VcQgTdL&L*as1R{GD5hl~1$A@?nJki*%vMjn)8YQ}uwa6uq z*FsRWU_8sm$7eb+J=XYdST)%B&z=(Atx`6zj~>b9ypi3-&~WvJ;nBPQTUw;Ce$a~@ z-FV5StUToA;UUdCzRna_MWGP1t_)Wy6eNZTT{CqJ4f2u<0UI(s;Vi6WY3M3%M}1Jx z($W%-YiP3zR-z8_3$y)p&{9QC3#I z0VgK%#z>Y0=~G{yXlK!Y!94|LpqyzHL)Hic1h|8^b;LU`%b0{>feHxQ!*}-Rj>*mu z)>OW>{_?H`X`+jeR8Q{IU10hTb7IIcGcgHE4_u~2XWgiZEpx*zdMWph;dGDA{qR3@HWUTR=#gLQuxZXGOaYzlatLR=ZEOl;F!23MadcHwzv~3$ zK~1#4{^?@9`bvKEpLX0_)Wi-|e@;k7pC7l++MyeR<8Q{H!2EjZJMSDm>9joB*@J$I zd4D}xFF8He*F9^lJuKh9aC*n0)_%cN#3~Bp$b`#wcByB5DuebTMz;N`;mM zNgdiRrze3^h8#Q$n59(_U}0q?xO?|z!U#-AFbqEpSVLJ0#pqRZbaeU351k1Go0mrk z5elZM0X{W1n-O5pT30@7v+Fr5?AI4bASK_+O6I4hnc9{A#QxK5Jq=+mpr*D_RuW{) zOVWhKy+VBi8gol)Nj}u(QuqpFDg~>tSm3xo5REm zcI5xduOl;_YLPupn3!G}88IE9tN_z+L`*zaX&(l+;Zs{96 zUYizcw*-CHGBII`=TRy6Q>S2)*4Efaw$FV)lTc4N`WzCrb=&|5<0^@yJ*dtnYJJ%^ zmV4?uuz0Vj&s^|o6#E9Tm=@KkI9bwNd6x*HBRV!VrZ{;P(*Q5q+gmZt)9q^YV&?vBFii?lJg-_jtc& z6|5->Jv_^?nBhZmaDGCbJ=>4;B_m-0+Au;f$V6yOVrWcID^?xyO1>&@2z>{l#STbL z>$Uk()qNSGSsr6uIooQKnvHXh>;S7UarE;Eva+yfz(zq(4&vC;s(1rp0~~l3wYY)- zOcCJ5sOk|I-Wi6%g&7nY9=-|C#g(zr(9@$MWHe5NkgE4@8cxVX%)lZGJ)3q}OI-egmbc*u zad8Evg@j>2v=V{i&`@g*mED;3!ib6;fZOmKMia-0C_tN_!_oTS+(?S8S#=va-LUQ@ zFx=w89vHmd66mTbl6>k084U?!Qxm2SIv)`rE!@gJgx^YR2w4c2c$ggc-!*q`X_f4j zlx!g+ExLQo*v+4(x$PT}!x%l=j+#>jptTHQ;#~G|`uOnnw?B_)`O!PF9O-cf*$3@X z_wiAP*u0&Jnwol(*y;vKPOjrd#SB!x*|Og89OFhb7Lm1083#g3FaJ3sBqHLHqZSam z_R0T<*egeh(tEdWb3qb9y2qo-2?Jmo>|Vx+MV#iP+h! z#;@!g9K#S?7!U2%4_fA4X;GNkeExm)dckVIp$~&`Rq%r5Q>K+yTlChor*-G3}LgLm50bDJ$Myj+C zk&$^%{v9~_Kg@FElOMIZaPGR(umV5VD}^ccT1IIJiNhspsVaV3(U5%K|9BAw921yK zO5Nc;fQr4ldr9T`oXQc5|6c0t?fsb}CCh`O=$<&6UDv8YdNJx{B0;KOTd=BKFt@OX zgcm?4X_!O46Fs*^CFCdp^rBM(wS|@IH@1lYoqQ)^7;FJs^NLE5#>0WDOEZM0IQMZd zi;cbk4pQv`s=p!*gUY5V_Y-}Ob8S|z;)BRBI)H8MyvW4Ulpox7plKvKJPPiF z8e4G$1A8m&E1i#?Qin)37?FdkN19&r5`wxjekvfV$t@xzc)PWhjqEAhFFHmygTseS zO1H_H{KxBhj>|gkf;+<+pv)7H_67ZDVBzlBRQ~Ekgb#{&6WkIvcr?p2P<|UT1;OX9Fdu1yJM;5>ZHLm3R zUz`yv&XUoBmmMDWuEi=$8LWzTU{ogq1EQf+o140F^oLMAH_>mkgI9=Iy+#y}WXbCc zBs?&3*k(vObLVNJ`bPGsxv$Gigj3cIt z{v$h0H838=jQjEZA=u4~mZj1DB-#elUBc@sef+rj@1Id|39DLPc|}w6x?fx8s36CIO$|6HP!j61tN)Z-l}<|A|8J8JPr( zIvQ2cxjJ=ZkYN`2^79uj_|?>SrD~?hiTMH0c9^pF|FNfasDl3qA=_7CG^{Wogvivr zeh*lanEtgVWYvkZiubFC7oKN#dipm`yCdeC0{@P1 zkcfT?XpVT%u@fg^K|F~s^EiLL>HeXZIcs6jU%JCW`}T3bwDIur3ahQHW!cH%lq#H_ z@JRX@spi7y76Ns(wY69Oq;EV2g_;@g`)z9J4(A0)uLxr|0%i!&#oE5l%_QTbaft9R zs5S!#pT*-daT=y9PAL!46S|Hr>sz>P$FDA6I*}E`gH?k@#<7DB0~*2vtEQXSzkh!e zCQb4;YA;S&&U}lJ8m}-jj(;$$NyCql5tf>oTBk46rD)I+07DeBQ!LO^QCKn?=F0w& zMv30}S-P9PqF0YG8oz~e-d2Hubv_GyXTvdph4~hIBY|IJ>cmu54>%MvT+jD@Z?l8$ z#3!A{9LT0S@hcA>N&pF;gY>U2G@~o}Boe@tr-@r&O55Y|F%L@{_cY~$E?gz}x8;gtP3J-_9 zH2L85+3NW5d^JYbbLWnpIPvh$wq(2W9F30m!BvP0MhmRl)yc^T&E=z|VgdqPBBB&b zB&~SvC`5S{3@y+XqHl!XpFrn?;jobrv!0&bhaf32F;sVfuB}cK9*rTJFx)^3N&`_5Co*VyHrq=kZw^rr2DS@ zp8q}LjB(DrAMZUMj=m0GAK1^c_xi0h*PL@L!ftCQ6XDb0BM3sIs-mEcAQ;W?Cjb{4 zUa4PfE`h(Wt!^qSAZO_Bw1(^#2*Ql0D#+@(y;+@bH_;nBl|CK9u#Ts9I_D%qgmoT^ zll(iSsv_RaZ4WJ;9y}cNI-i~xs;sKQ>t0X3+^LBbwS8bkoc`**Ee}t41{cr67B4d^tM)m} z*wE0-c7%FM4SolSQFVp&KcY<)v_e*o_-Jrr2>mg@lB@r_1BO z^_N;7`9TEOv6+*J64ZkHF>&ZlL>C>#t1_=*GcQKw`lQ>9mg6}&Il=oYV0S+9vljik zwPpF>LGI2v`jsi~J<|l1{rs`a?72qvtWDh5p5TIlD-Q=h-rF9_|I*;Ow>=pmxP2fd zziOPZB!GNQd-CK7DFp>4H8r)Nv2j~R$9W14_2#$Yt~>=P2?;^P#R6htVrjAg*rAb; z?WWk0WaLHXYJe)!&vRH!_ygc5r>J1>5YUtIj=!sJA9zi_JWqg z-uP3*Z+zSY3mbcOvcWS+!d+;z{6WC4U)F>)0y1-_lU`f59z#sqf{EIiPIveRKI##S zjEtygXeim+^U;Yu4sCB&xNK5Qnyv9x&dO?5r|Mwg$0b5~G14jDGf8Yh8owcPUxwY? z-A|uBHUBIy7Aahv?TEm|!;`hPW(y`}2)}sEJ}oB)mxYBz<<_l^g&*%{R#&4^t~pqZ zl-^sIY!Huz=)iT-Tc^1XQeUs3&mJy1_g-CJpDLJ;_PN%3B^DkY{IipFUqPF1^j9Ab zla`c}(4Be5W@5!*o4L7){2u5_3(v!7U4Sa^7pJ8W-b!+CsilDhjt!PYh>XDwB(!;a6eSOy_oZ#whI zdUo^X&D2qY5mAiLuPN6OADI_h_}|Xeoy|`5;Td|Dkih=f&Q4KR_hs#^v6J@^QbFGz zr#`Hm;{BQXb)W0VJ3);-no;tjeVy-FBOH6-#_n$6z*NoCKY>9(K{VnnD>{!ZaXVWf z@k-8Qevot2w^Bq4C@I<5*)vpMaJ2UK>rDLq{hPzY#DvFg?$_rRbT}4UzrQNNIdyx> z>MN!o+q=6|(b3TvIXMeqV_xJ)hF+e2!l<{mcTiPT)u+#&bGPT#*ZCo9KA0K&S2xJS zl&UtiI+&MOTvnDh@D0htzp+WjOxu&dYcRdIcxR37B4TK0m?j@WT1Le7r= zNGpwHa~uuC9yM4Ql0OK*P{E#CS_+4{l*QFri^wZ4r(9fIgfiBvs(3hvPt9kNQyv9( zS8$kbP{bHa!W2Ez7J`XDmP99-`_K|lU2cW|BDQ<%OTT{y_h)O)0rK5w3TiT~bNTf3 ztD>XhhR)F7;OUPKy+Z$1w<;-{vY7;EaZ)fcMnj(K*NzEENikO0Pqf0b4BBDh5V!9v z55>KFscS!xbu;nd_|!z5D{sZY(GjLhO}`Tapm0F{ZQLbg)@#?OnS76(vhHOBS4}D_ zDY1%+UtIoKkoV+ZF6P{V)tT)WXWEF=pmUQ!=GT4ZzVnP2@fxPJ&Uo;+1y4o-@hFPj zx9&$WO44y^yzxs+q}F(QBM45n_3Kw6coc&oaqmOt$K%yxZ6PGek00kBik|iKs3Wf# zDnuM6=}(UKLYkVS=|mm|0(`W0b;)OGLnW#;nEm=$8VQn-=6VJc?LdbNRITUGj+ z!z5*u!xYtAXB35ohDPg`FKS}q;t@$nNjz04$;sSu^726q4Gn0~(cEy(`Sk4@6`}yY z5>BLj3}(dF>o3(lTjqdt=Dd9QvO1%8A3b_Bv$n=(b@#5wXri87*=(h^msjB1w{MFpE2#<# z3t5GPv@4%iep5~g3c=^2{dv_|`SdXouln0$ddyF&H>m0H$R#lYVEqG&G zT0x`@8f1BEYwP#gXOEoM#%Vh`I#kruTA?yY>FFb5V@b2Rb?EmoxEK}| zrlPK{;N&D|RBr3VE65bZ9PGS?97QTN=$>?A)JM|@>f2a!QJ_k*oimED}+Rd8@eDe6Js;aoe z#K91vzN{p6C|5(njU|$nL<|*c>+$jNjnPL%mcM(F)e_~bzwPr}Wxe}k{>ugH`}eK7 zUtK|8+@wT$YWgvs3Eksj{@hAQrMk9P~wc zahLq@z}gEoN}Q|{RtPz5Sh#=Ut5?BranMH=AzxrvmjJyC=eac*&yz2bz%{ri#JH|c z77R2#=;wI(@+H@8vbcl0aIODR^tSzEeHipiOapl~9Bk&e*3Ps$m!^3JTirTx4-CAu|6^ssDNT|4mHtU-J3oWB^be1E{GbrYF65L!X(Q{T|Rx zEzyb+`t|opJ7ZwE!kOf9xLFa*9^>s{6eU(2_$c!ge72<9lPYZv1o?RXFK%q4_S+lJ zivNvP2kop+nJ3=rrDlVVIjwJ?Ylg(TRKY3ftJ& zn39r`OE=y9b8l}*R#sNuJ<>(O8053{d2H+q;7x{2KHgdPmWPT$-oL-}{P}aroF0jh z(a{Y30wcJeOkqM9p*uf+{zS<#z!lU{jy?b7%k4l$qk#qTG!+xz<8QR!Ff~t1FaT?| z9WG`8hU>UF+wo+10!jh8Ku~Hb(>u?#>p;if18<>`_K^Sp=CP|FH@dY$7^lAt7>TAHORUwFxm})s38;um1h} z_lu-(^|NLDU-R=UH*VDS9RlP7iEXKO-}+u;E}d^uLk{TE26QHJZK7`OO9JoGGhrd2 zV(%jttn=qBXIg^{ORUbz$;lPlPu#%5!kPvikx-cm;sQ{TSxk)P=BvxHW0iI&hrN8K zR3WeT*jPtY5G@>N{2i01*BqUwX zv&cwE{f_q5rTc3~`+f=k`Td(!K!6lp=mLr;=zHn~z-7_whtWIEuN8&IdP&%_6>DZ; zVZZ$u68io7ZI(dN#>PfgHa3NZe21yV*4|!CTo#o4x_Kq>1M|(%<=O1lluj5k)7O7h zTv2gBOG|615=5Prk&#jEdSDKHPQhl|t4J^lkDAI&$`mmfLxHcH`0Jy@-pqC(n% zT6UlKI6FHhj9t;GbFtgp9dj_KiOdPU0U}S`4327dZ6X3B#q{CM@^F=dd4jSpQVe+Q z=I-7C*;Z3ivy^({*-{#$7)TiXB52iljRED=)imzz?!eeOBKR9|5izfQ$E;OIu=h6; z`Ie-`LHxl5pL%)*mV~d1i_Z-jm|q#%{b}rgfi#}%8Y8|;i=rSr$F(d+PHp=#@ha^n zKHB%+H?@24H~B?;{4^vTh{uGZ31C2yPn#4(I_hLT|2o-9J!>z2&`-+5M8bjv$tmcy zzri!~_0uOAA0H_oR=V?lyQUw45Rfbks^K5;x!FX}+?VC3|6@!JQ z7Y_~&M!nd|;l#94LPBtm9L;{SnkVNaJ+^bUlR|=m&fjoe@O$}^@>z=L^23`idcAg* z^g!_G8x?)*Rkh}VdUXfNsqE%feFJH0Ys-79nhs3Z$H!;C*Y|X5VD)s_{H!%YF@lte z3Yew`0Z5S`;AMfNsE&K^;6c_@+-9;ot+a45G{5Aag#}x$gZV_n56Gl`omrjBYHUhM z1e8~{{ZkME3j?_?4qx6)?s9nfBQY{EQq2xXAtq98KS75~|2rG!3j}Tu_wF6Bfq?;f z-%Ij&`UQ?qNNTHk-Wsg(Ga@oV1T8HsNy0Y7zrVeWkn%p%tuq5rSz1@wELMe6+xip5oBg&rrLdrgPek*yMUUBDF(EM<;HZ& zlBnI!!4R+>W+9V+F7XUCIxlKB?B3jFgMM3PVQJ}qr|iDuvxZ!qY^&aPk~ESYS9e!N z0-&R}b#`JK6q>XEk)58O4-$Fw6F~sq;Fx1DFtM30UnT?*xBolE0X_!P+m~MA?0Abt z_&y=3bSv$~toGI>hD+}epqG%6B1n>y_YF~p$w*KZbI>V3IJLrO7C)I+tFEcBJltMf zT76JfD5ah%#Q=%lzA@8AcJX2avz206KcMXXQgbwOUS_=mBQX+yTBAcDi@lO z!HS~6%-r0im6esiii!#lJs0BQ;#k?)6&ohaeNWgws3)_6xZ%jp&sP9t*^#OGq8;Q$ z`sUqpgM(?GZNwd?{Xn4Cp)ddmaRC+3T=1)E4b*B!wbNYSix)47OH0YWef!4B!J+i) z*Du}CLP)^FgL2T|$;qr&uj&qujYR<44o{a4313-xj3EO&T(^dQp9X7Di}M?&VvLNFPmFxUyG6~o2Hr=+b-4)#sZ^t44_U?7(%qiQHn zC+r_}t`2M*9IS$ZWI&c}e`K7WbpON9^G-75T1zY|$~8Mp_`7?P1Tls0_uQ@}WEC;P z{EzmW5DzpR&=s8lI3(ro#7XK(?2e-+BS*$6e61WL~mIG~u*EEe==e^e~iyvxdBH}hN-0Y!7My4B_>!0XCX z6BB>aaVb#ew*bY^KCqxW<(Srqf_Xs(6d#^$|KNbzlrdQp*t`CdCqfw+85>9IO$K$} zK$HtU9N@y#*Aq@pPv76{V%pr8k2`&La8?<4kh3_o;`}+t|>$*-1{^qmgpDj~U z<}+#)!cdYB$@eB>i~ZRbJ}NPuo$fUaX7{eHti;C0Ki`}5Bmvpo1*GG)k{3p*8Hhn8psXUoa?&V9!iF<|FI#@plEh3M!FD;oB?TI zcyLJ?gcBSw#QtTs>ta@MU7Ko(0RXiGk-optMzRTT_^q;En~Pl-t$scR$yNH6DjgF>!8yA(<=gdlzWYa+jb_$Op`@`Yhc;jr zpbpO?nIAr&J*G2+$tTD0p6-1xWPpCQfKMyvx%=4A$cV?3oR)+Uqpysci%Z7DgdPaA zfbDl?FlMzsZeaTT8>(rY*houDw|8`K8<6S=xoudyx$z9YE1E&x!h)I4yfNQd)mj10 zz5;wEfKJqb6ViAm5jH$*_@XXOky;~F%9IbmNE!$@E=386%gV4n5O+v3rd03Ub+$1q4Rz9=(P@viAnyFH1iJ3VNrZmWZAoFx!Xof~bfsrM^dg)RL(Aj*W z3Vb+ggW9Xwq%p{1Zg_Y&DK#~Yxw$!D%nYP=UkKBNE8^kq&TWeQpryZ`LOGh=KQE8h z%iG&&V_G)rChPf44jPXPFn6q41MzJ8KM=j*GYyA^#bp|=#dYP1B1j&`t>3p$Z(1!; zzSjcSj@w++wT&;lMHXPQNO)gin#7XVt9!Ogg*wP!8(Kn&4c9#7Y+DxuVPrdAE!fr@ zcpe7_ZJ2<=N-{Etkg#w9Ul<|O<2=fNp`@XcJ_oi0a|bm3S=~Z6vNcnGFKP;Z&(XT{ zc&w6$f>T35M#fL|R+pkkSBizoD)d%VN`UGiW|Y8%LWjdQvibHV*BOT^xiBY(%X#@H z5j0UT$7w{r$c#u?SvjklRdSAunFYK$;397G2?hedcG);Ng^%Ao$FTJQcf;}5Co+_u zj*VRucU{x1W$P2hPHX?dk+ZY0A#-xL18?Xb8j1@yh^Hrssq&%s%NKkYexMwIL-SoE zw8Pn=rc>?O%Y78sX_+XC0QxfnM79|i2#Tl!c?Oz`rRuFv;hRv|>5w#Fnt05vugAb6 zPQ%4=fZ|R|1Ke;pk`{>CZ(uQS>K%ah0x+cjuL|YAYKix>zrB+raCLPxyt7GOdp%Dl zn;3Z0=A`GkC1{E_jzffk2>a$fRz>ra`RoPG9QLmphp8s0wu49939M^6JVw#)it6yV(Sc+FF2?SbVK19az#&a@QoQLHs?c3#BA#g(zL z%78cp)A(_K>-31`sx7INl~u9r4^}Ujve{17*T4gdn4P`<2!bq16L4aVYcQLNFiNYUE0`y9+uL&s?ba~McS{n2g9`cvgz9{4nWe}5W5 zD=Z)A3ox_^h6^wlG3e4u@g9O~L1}NQ&tce%4c50e*V~$KJnk15S9Jo1pI%)h;^*f_ zM{;V3Y?9b%WMpy{J4=J}969rmM4bW3e!4mkeV?tb*Pw^vbB^oLV18G5bNwkcs=h%0 zppU0<`;j$h`w(!ut7~g{O!1Tb;h&d8L@t2*T5^qupAkeZ=!?qQsi@$hQ!}WV=Aj{j z#nPPTfaxI47^$d^Ipu4$vNaf>`!e(KMN_Qt1fIRV@G>^m50vfuv@{t>*(*&JQ4~dI zag8r;Ro6Wl;?t;pWN}=nC!?fdz zUiM%S;)4$AioV9SK-v(^b;bM%tUGLR^v1!h#_X99;j4(Pw{r|g$mUW(_sw>lPr4rL)tZ?qx<{`g$qGor9b?{@+xh*=g_9^ZXnwDUjV1WG6tG)cb>yiu+)FKWvn*TsGgHzeu zFbR{xLG{daG5IP0QcW)}cjLK=G$UT;Sg^>+$w8)e;5}sT&`Gma6Pqb?q4673(bPnd zuh{G0$>XU;h6rkYC5Q>+4F-uFO9_V?#vO2EK@irFCMnPo_`7Da~=Og{S``bEVl$cRl&Xnf~%mtpmDatbguaEh!9kZ z@?!iaT+;@R3xF32E-saQS)_|!0UZQkkQGTMqSlG$I`2+xtEELYJ39-}CjfE>gfD(Z zd60*jyJa9(CxI_Or3fl1B-Qtn77o5+FwY=?uWLH<`Z)<;6VPHMFPk^g19$@^Qc(Ev zSMM?c6sJ|tE2Pn(z z%3|JdH|oIloDydO=VRgJy$M$uLF{}^4{X3w;5H4Pq!)8UV7dv~R)`fOFwDugP>Kz* zI}f7Eb?DfbQ6@V)GQvI5Fu7T%Jpt-TaJi&4Og)$9)vH(cCD$;xPG7MPX+-D$2990EBoeP@=JVM0e^(&W=HzI*i~cV8T8c5runZcb-$ z@6(2unTmcb{ZVPO=ZiOFJ}@hxk#G}069LTJw0qsJs$JI^QRV|Wm>dTFj!sS(2q<{{ z!xI>kwsv*lfJ=~9Ndm$L2F)|5jk2?IrRp+(H|R8QFcf8F{q3dc7wNkQXd~)e*Jt5K z7GU-*YgS&gIGpG*P7nm+Al(sSnZV#+sZCc&Nl8VJE*X-qnA>39^5=ig+3(o}pDi=7v?Jrzr0AptOjzlG{>VGyBb*#4JzEf&r#pdkOzqO^CuOwxjZMP|DzqRGC6+-6E-GLjRK+Co}}*T z?iaEBj(`Oo4D7hpWi=e~oRyPPd9k#o59-ws#_e9)Ka@9sohI^`7F&NNZ&-2gj`7xE z+T)=0gZAGAr-1gnhFuByL|UweF~g9}J4j<_p6ZqsbSsK*uqkp&fpKRG$MFkT}J5<430(%`U1(TVikk%rgd=sTd&1RoCzAkFYU zuo}?y;K{>J;K111+gCfzoI_qh=GegYIW1drk*`?_H64 z>y)dPM+M^PONC@(Q$c?I^vX&kyl+ce+g$RIVEgd3c_)W>jQ!4w4$Xsu7XW`dLAatT zJE*y!udk0D7}#FUF2f8EuG7hxMExh_Mc8vD~Z7P zaM9w-kugi-e5cK0{Kt_ZNVW|jRQ zBn6Z{fa|bVAP4p}jEXBp&7LW*=~WDEb4|TuV`tAj1F8EH{kpC$rHQHO&S;TTm(Q(6 z&IIef82i;gGTws+0J|?~8dVn=7z(8Bm4K=B*4|hj#EF>8O44C3SVO3aP>QAx11E*{ z*EhlF#J^H!$4?QN7aX?+&M>t%NBBM_gDsqITt)0qzlm+;d-4os1U!FZtEHg00C)+} zjW~UGUDP?813yDV%2Zzqs2x-ZW`bIWs`E2zh@IQUG{(7e=LDYoQXyoxq01R_NjGS2 zesOKGAp)`yikbm+m@Mwyi#I1yDk?4}#UZBm192gHDEw>qT8ERv%lF4H?@AJPwNL1t z^jN+E$(8}Ul19vt8QqApvx|aaV1;Q238O?a>az5Ez@CP{qo0@G+UvN0^Z8GXXdF4# z510T&DE8QS2+hy4S90%-MwN~F-6_hh1}*IAAOdOu&)pTC#D;_4DPT!rqApWrCL0P$ za5AWC0?y0^446=_7`rg3dWlWOL&hVaNY58u6oh1CTDShMYn$5V(+YnNvu<@d$qB_fjq@JTTNoN9$sUP=MJyp9 zLAblt0F?<-MA}ULr=l>ggIz?Ld};ZZ7vL6vM*7C=9k)1K=UOD&bcqOUyi(cktuXds zU`AKo1b*=G9MhSMtu5Mu2!9rF_ujcrh_>?aCf_NG6K)X=;GRT3scD-Uy^rXeT}ZP! zPXOeuN~Gv9PtZt?0gd<%%H4c1(Tw&hURenZMwD{RS`iK4EWU8 z`)Iu*J(1Rl3Da`hA9D3)r$<&_UyHCZGsD7ds@LDJLh0w}&O)CNO1V(5ER7ISSkfl- zJ=saz*xsf@H}b$581gWNm8%SJ;`R7EH#?|1p7u%_^gmH$sio(<0DCiriyq^cu)6o3 zmJ5O6GM?$I1++-0_FcLjKtOD~7g+6Xjlka4`!;fKNG5Xk|FGif6IGeV_JX(>#%U&XE<^}A%7~ATw;7Et zD--rP*%{(VoTzrf1SIM;xx}q$Fc0_hRzdv-J>=Da{U z>3eo!GpY}i9X4$gLHTBzcxq@6^O@ELg4$Tlu-5kD>%xo6ovt2=?c=me9fVDGbQc-c zC{kf!4Aclr1%ge$^@j}wzy$CSD;RjskAnZrjf8^DvzU2bNWwJwMR+qT9A-L8uH{`^ zh9qtcyxV@&;rX39mjZ!SJ+_3r2j{Xh*F7SyCR*z`x#Pj1kCFNF4otp%8x_*wGm6i73 z^~yiUD`NSRzwH#tDYaGum?+V0JX+y@VYX&S=I5up_m4MK< Date: Mon, 12 Jan 2015 11:59:57 -0500 Subject: [PATCH 145/417] Remove mentions of 'uncompressing the data'. --- doc/lstm.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index f78fb5ea..550a5a4b 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -26,8 +26,7 @@ While the dataset is public, in this tutorial we provide a copy of the dataset that has previously been preprocessed according to the needs of this LSTM implementation. You can download this preprocessed version of the dataset using the script `download.sh -`_ -and uncompress it. +`_. Model +++++ @@ -185,7 +184,7 @@ The LSTM implementation can be found in the two following files : * `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. -After downloading both the scripts, downloading and uncompressing the data and +After downloading both the scripts, downloading the data and putting all those files in the same folder, the user can run the code by calling: From 84a4c9ebe9afc1134170bba44e0a04ba267c9611 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 13:07:16 -0500 Subject: [PATCH 146/417] Add reference for forget gates. --- doc/lstm.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/lstm.txt b/doc/lstm.txt index 550a5a4b..f929e84f 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -200,6 +200,8 @@ If you use this tutorial, please cite the following papers: * `[pdf] `_ Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. +* `[pdf] `_ Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to forget: Continual prediction with LSTM. Neural computation, 12(10), 2451-2471. + * `[pdf] `_ Graves, Alex. Supervised sequence labelling with recurrent neural networks. Vol. 385. Springer, 2012. * `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. @@ -219,6 +221,8 @@ References * Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. +* Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to forget: Continual prediction with LSTM. Neural computation, 12(10), 2451-2471. + * Graves, A. (2012). Supervised sequence labelling with recurrent neural networks (Vol. 385). Springer. * Hochreiter, S., Bengio, Y., Frasconi, P., & Schmidhuber, J. (2001). Gradient flow in recurrent nets: the difficulty of learning long-term dependencies. From 4818f503a3631bf086d0ada95f3f0499cff85478 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 13:11:06 -0500 Subject: [PATCH 147/417] Add a description for each paper we ask users of the tutorial to cite. --- doc/lstm.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index f929e84f..c9aa95c2 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -196,14 +196,22 @@ calling: Papers ====== -If you use this tutorial, please cite the following papers: +If you use this tutorial, please cite the following paper. + +Introduction of the LSTM model: * `[pdf] `_ Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. +Addition of the forget gate to the LSTM model: + * `[pdf] `_ Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to forget: Continual prediction with LSTM. Neural computation, 12(10), 2451-2471. +More recent LSTM paper: + * `[pdf] `_ Graves, Alex. Supervised sequence labelling with recurrent neural networks. Vol. 385. Springer, 2012. +Papers related to Theano: + * `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. * `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. From 980ab7564cbcf7e77884700d33f9de1b4de964ee Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 13:16:12 -0500 Subject: [PATCH 148/417] Typo --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index c9aa95c2..cc0c1f07 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -196,7 +196,7 @@ calling: Papers ====== -If you use this tutorial, please cite the following paper. +If you use this tutorial, please cite the following papers. Introduction of the LSTM model: From 8cff23c09e83d8201581075014acb2d086cd22fb Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Mon, 12 Jan 2015 14:55:21 -0500 Subject: [PATCH 149/417] Remove references to downloading the data --- doc/lstm.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index cc0c1f07..624691f6 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -24,9 +24,8 @@ recurrent neural on the Large Movie Review Dataset dataset. While the dataset is public, in this tutorial we provide a copy of the dataset that has previously been preprocessed according to the needs of this LSTM -implementation. You can download this preprocessed version of the dataset -using the script `download.sh -`_. +implementation. Running the code provided in this tutorial will automatically +download the data to the local directory. Model +++++ @@ -184,14 +183,14 @@ The LSTM implementation can be found in the two following files : * `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. -After downloading both the scripts, downloading the data and -putting all those files in the same folder, the user can run the code by -calling: +After downloading both scripts and putting both in the same folder, the user +can run the code by calling: .. code-block:: bash THEANO_FLAGS="floatX=float32" python train_lstm.py +The script will automatically download the data and decompress it. Papers ====== From 8afe749dd5373b7378170f960215ee1414faaec7 Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 12 Jan 2015 15:40:58 -0500 Subject: [PATCH 150/417] small update --- code/lstm.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index d23c6e76..200b359f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -291,6 +291,8 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): def build_model(tparams, options): trng = RandomStreams(1234) + + # Used for dropout. use_noise = theano.shared(numpy.float32(0.)) x = tensor.matrix('x', dtype='int64') @@ -378,8 +380,8 @@ def test_lstm( validFreq=10000, # after 1000 saveFreq=100000, # save the parameters after every saveFreq updates maxlen=100, # longer sequence get ignored - batch_size=64, - valid_batch_size=64, + batch_size=64, # the batch size during training. + valid_batch_size=64, # The batch size during validation dataset='imdb', # Parameter for extra option @@ -448,12 +450,13 @@ def test_lstm( if saveFreq == -1: saveFreq = len(train[0])/batch_size - uidx = 0 - estop = False + uidx = 0 # the number of update done + estop = False # early stop start_time = time.clock() for eidx in xrange(max_epochs): n_samples = 0 + # Get new shuffled index for the training set. kf = get_minibatches_idx(len(train[0]), len(train[0])/batch_size, shuffle=True) @@ -462,10 +465,13 @@ def test_lstm( uidx += 1 use_noise.set_value(1.) + # Select the random examples for this minibatch y = [train[1][t] for t in train_index] - x, mask, y = prepare_data([train[0][t]for t in train_index], - y, maxlen=maxlen) + x = [train[0][t]for t in train_index] + # Get the data in numpy.ndarray formet. + # It return something of the shape (minibatch maxlen, n samples) + x, mask, y = prepare_data(x, y, maxlen=maxlen) if x is None: print 'Minibatch with zero sample under length ', maxlen continue From 3d9b1ac652a9e26fefd2219644c3cca94cbed42d Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 12 Jan 2015 15:42:08 -0500 Subject: [PATCH 151/417] fix the display of the number of example seen --- code/lstm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 200b359f..e91fbd84 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -461,7 +461,6 @@ def test_lstm( shuffle=True) for _, train_index in kf: - n_samples += train_index.shape[0] uidx += 1 use_noise.set_value(1.) @@ -475,6 +474,7 @@ def test_lstm( if x is None: print 'Minibatch with zero sample under length ', maxlen continue + n_samples += x.shape[1] cost = f_grad_shared(x, mask, y) f_update(lrate) From 16bf61c98fee16e17411d115b8dca8b2f58ee308 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 12 Jan 2015 17:03:54 -0500 Subject: [PATCH 152/417] Bump the recursion limit for rnnslu. --- code/rnnslu.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/rnnslu.py b/code/rnnslu.py index 4ab80b54..65363688 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -15,6 +15,10 @@ import theano from theano import tensor as T +# Otherwise the deepcopy fails +import sys +sys.setrecursionlimit(1500) + PREFIX = os.getenv( 'ATISDATA', os.path.join(os.path.split(os.path.abspath(os.path.dirname(__file__)))[0], From 7bc712b06a9aaa62631230fe2df794822783dc26 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Mon, 12 Jan 2015 21:10:06 -0500 Subject: [PATCH 153/417] Update timming since speed up --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index b366dbbc..df109332 100644 --- a/code/test.py +++ b/code/test.py @@ -105,7 +105,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 21.5, - 11.8, 47.9, 290.1, 315.4, 72.4]) + 11.8, 47.9, 290.1, 255.4, 72.4]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 401a99a00ba77c70bdb5ff04e665a7f9978622c9 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 08:40:34 -0500 Subject: [PATCH 154/417] Add a way to reload pretrained model --- code/lstm.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index e91fbd84..8a7e42fb 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -80,6 +80,9 @@ def _p(pp, name): def init_params(options): + """ + Global (not LSTM) parameter. For the embeding and the classifier. + """ params = OrderedDict() # embedding randn = numpy.random.rand(options['n_words'], @@ -125,6 +128,11 @@ def ortho_weight(ndim): def param_init_lstm(options, params, prefix='lstm'): + """ + Init the LSTM parameter: + + :see: init_params + """ W = numpy.concatenate([ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), ortho_weight(options['dim_proj']), @@ -388,6 +396,7 @@ def test_lstm( noise_std=0., use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. + reload_model="", # Path to a saved model we want to start from. ): # Model options @@ -407,6 +416,9 @@ def test_lstm( # Dict name (string) -> numpy ndarray params = init_params(model_options) + if reload_model: + load_params('lstm_model.npz', params) + # This create Theano Shared Variable from the parameters. # Dict name (string) -> Theano Tensor Shared Variable # params and tparams have different copy of the weights. @@ -561,4 +573,7 @@ def test_lstm( theano.config.scan.allow_gc = False # See function train for all possible parameter and there definition. - test_lstm(max_epochs=10) + test_lstm( + #reload_model="lstm_model.npz", + max_epochs=10, + ) From 6b7b7a6cafdd56bd541a3fc14b9fac10a0380600 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 08:41:35 -0500 Subject: [PATCH 155/417] use adadelta, sgd do not work. --- code/lstm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 8a7e42fb..ba733c60 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -375,14 +375,14 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): def test_lstm( - dim_proj=128, # TODO: What is this + dim_proj=128, # word embeding dimension and LSTM number of hidden units. patience=10, # number of epoch to wait before early stop if no progress max_epochs=5000, # The maximum number of epoch to run dispFreq=10, # display to stdout the training progress every N updates decay_c=0., # weight decay for the classifier applied to the U weights. lrate=0.0001, # learning rate for sgd (not used for adadelta and rmsprop) n_words=10000, # vocabulary size - optimizer=sgd, # sgd, adadelta and rmsprop available + optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decay learning rate). encoder='lstm', # TODO: can be removed must be lstm. saveto='lstm_model.npz', # The best model will be saved there validFreq=10000, # after 1000 From 6a3f6d971126766e8a2f96090d5f2b3dbe681ac6 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 13 Jan 2015 08:46:27 -0500 Subject: [PATCH 156/417] Fixed typos --- doc/lstm.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 624691f6..c05256d9 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -19,8 +19,8 @@ is positive or negative. This is a binary classification task. Data ++++ -As previously mentionned, the provided scripts are used to train a LSTM -recurrent neural on the Large Movie Review Dataset dataset. +As previously mentioned, the provided scripts are used to train a LSTM +recurrent neural network on the Large Movie Review Dataset dataset. While the dataset is public, in this tutorial we provide a copy of the dataset that has previously been preprocessed according to the needs of this LSTM @@ -40,11 +40,13 @@ matrix associated with the connections between the neurons of the recurrent hidden layer. This means that, the magnitude of weights in the transition matrix can have a strong impact on the learning process. -If the weights in this matrix are small, it can lead to a situation called +If the weights in this matrix are small (or, more formally, if the leading +eigenvalue of the weight matrix is small), it can lead to a situation called *vanishing gradients* where the gradient signal gets so small that learning either becomes very slow or stops working altogether. It can also make more difficult the task of learning long-term dependencies in the data. -Conversely, if the weights in this matrix are large, it can lead to a +Conversely, if the weights in this matrix are large (or, again, more formally, +if the leading eigenvalue of the weight matrix is large), it can lead to a situation where the gradient signal is so large that it can cause learning to diverge. This is often referred to as *exploding gradients*. @@ -55,7 +57,7 @@ connection (a connection to itself), a forget gate and an output gate. The self-recurrent connection has a weight of 1.0 and ensures that, barring any outside interference, the state of a memory cell can remain constant from one timestep to another. The gates serve to modulate the interactions between the -memory cell and itself and its environment. The input gate can allow incoming +memory cell itself and its environment. The input gate can allow incoming signal to alter the state of the memory cell or block it. On the other hand, the output gate can allow the state of the memory cell to have an effect on other neurons or prevent it. Finally, the forget gate can modulate the memory From 7f067b8219d1499b0676167034e76831342a0e33 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 13 Jan 2015 08:48:26 -0500 Subject: [PATCH 157/417] Fixed typos (2) --- doc/lstm.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index c05256d9..8b04df54 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -41,14 +41,14 @@ hidden layer. This means that, the magnitude of weights in the transition matrix can have a strong impact on the learning process. If the weights in this matrix are small (or, more formally, if the leading -eigenvalue of the weight matrix is small), it can lead to a situation called -*vanishing gradients* where the gradient signal gets so small that learning -either becomes very slow or stops working altogether. It can also make more -difficult the task of learning long-term dependencies in the data. -Conversely, if the weights in this matrix are large (or, again, more formally, -if the leading eigenvalue of the weight matrix is large), it can lead to a -situation where the gradient signal is so large that it can cause learning to -diverge. This is often referred to as *exploding gradients*. +eigenvalue of the weight matrix is smaller than 1.0), it can lead to a +situation called *vanishing gradients* where the gradient signal gets so small +that learning either becomes very slow or stops working altogether. It can +also make more difficult the task of learning long-term dependencies in the +data. Conversely, if the weights in this matrix are large (or, again, more +formally, if the leading eigenvalue of the weight matrix is larger than 1.0), +it can lead to a situation where the gradient signal is so large that it can +cause learning to diverge. This is often referred to as *exploding gradients*. These issues are the main motivation behind the LSTM model which introduces a new structure called a *memory cell* (see Figure 1 below). A memory cell is From 4829202af610121c1a4db015cea3b38f8c34f6c7 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 13 Jan 2015 09:27:16 -0500 Subject: [PATCH 158/417] Add notice about using SGD in the LSTM tutorial --- doc/lstm.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/lstm.txt b/doc/lstm.txt index 8b04df54..c9c2a3dd 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -194,6 +194,11 @@ can run the code by calling: The script will automatically download the data and decompress it. +**Note** : The provided code supports the Stochastic Gradient Descent (SGD), +AdaDelta and RMSProp optimization methods. You are advised to use AdaDelta or +RMSProp because SGD appears to performs poorly on this task with this +particular model. + Papers ====== From 194adadb94c5571c12b5ad7de903612f1ff26968 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 09:29:05 -0500 Subject: [PATCH 159/417] Add the script that created the preprocessed imdb dataset --- code/imdb_preprocess.py | 123 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 code/imdb_preprocess.py diff --git a/code/imdb_preprocess.py b/code/imdb_preprocess.py new file mode 100644 index 00000000..c20b37b6 --- /dev/null +++ b/code/imdb_preprocess.py @@ -0,0 +1,123 @@ +""" +This script is what created the dataset pickled. + +1) You need to download this file and put it in the same directory as this file. +https://github.com/moses-smt/mosesdecoder/raw/master/scripts/tokenizer/tokenizer.perl . Give it execution permission. + +2) Get the dataset from http://ai.stanford.edu/~amaas/data/sentiment/ and extract it in the current directory. + +3) Then run this script. +""" + +dataset_path='/Tmp/bastienf/aclImdb/' + +import numpy +import cPickle as pkl + +from collections import OrderedDict + +import glob +import os + +from subprocess import Popen, PIPE + +# tokenizer.perl is from Moses: https://github.com/moses-smt/mosesdecoder/tree/master/scripts/tokenizer +tokenizer_cmd = ['./tokenizer.perl', '-l', 'en', '-q', '-'] + + +def tokenize(sentences): + + print 'Tokenizing..', + text = "\n".join(sentences) + tokenizer = Popen(tokenizer_cmd, stdin=PIPE, stdout=PIPE) + tok_text, _ = tokenizer.communicate(text) + toks = tok_text.split('\n')[:-1] + print 'Done' + + return toks + + +def build_dict(path): + sentences = [] + currdir = os.getcwd() + os.chdir('%s/pos/' % path) + for ff in glob.glob("*.txt"): + with open(ff, 'r') as f: + sentences.append(f.readline().strip()) + os.chdir('%s/neg/' % path) + for ff in glob.glob("*.txt"): + with open(ff, 'r') as f: + sentences.append(f.readline().strip()) + os.chdir(currdir) + + sentences = tokenize(sentences) + + print 'Building dictionary..', + wordcount = dict() + for ss in sentences: + words = ss.strip().lower().split() + for w in words: + if w not in wordcount: + wordcount[w] = 1 + else: + wordcount[w] += 1 + + counts = wordcount.values() + keys = wordcount.keys() + + sorted_idx = numpy.argsort(counts)[::-1] + + worddict = dict() + + for idx, ss in enumerate(sorted_idx): + worddict[keys[ss]] = idx+2 # leave 0 and 1 (UNK) + + print numpy.sum(counts), ' total words ', len(keys), ' unique words' + + return worddict + + +def grab_data(path, dictionary): + sentences = [] + currdir = os.getcwd() + os.chdir(path) + for ff in glob.glob("*.txt"): + with open(ff, 'r') as f: + sentences.append(f.readline().strip()) + os.chdir(currdir) + sentences = tokenize(sentences) + + seqs = [None] * len(sentences) + for idx, ss in enumerate(sentences): + words = ss.strip().lower().split() + seqs[idx] = [dictionary[w] if w in dictionary else 1 for w in words] + + return seqs + + +def main(): + # Get the dataset from http://ai.stanford.edu/~amaas/data/sentiment/ + path = dataset_path + dictionary = build_dict(os.path.join(path, 'train')) + + train_x_pos = grab_data(path+'train/pos', dictionary) + train_x_neg = grab_data(path+'train/neg', dictionary) + train_x = train_x_pos + train_x_neg + train_y = [1] * len(train_x_pos) + [0] * len(train_x_neg) + + test_x_pos = grab_data(path+'test/pos', dictionary) + test_x_neg = grab_data(path+'test/neg', dictionary) + test_x = test_x_pos + test_x_neg + test_y = [1] * len(test_x_pos) + [0] * len(test_x_neg) + + f = open('imdb.pkl', 'wb') + pkl.dump((train_x, train_y), f, -1) + pkl.dump((test_x, test_y), f, -1) + f.close() + + f = open('imdb.dict.pkl', 'wb') + pkl.dump(dictionary, f, -1) + f.close() + +if __name__ == '__main__': + main() From c6fdcff288103b707f640c3f19f51b50eb5ea9ab Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 13 Jan 2015 11:10:46 -0500 Subject: [PATCH 160/417] Fixed function get_minibatches_idx() --- code/lstm.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index ba733c60..1946c0ad 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -18,7 +18,7 @@ datasets = {'imdb': (imdb.load_data, imdb.prepare_data)} -def get_minibatches_idx(n, nb_batches, shuffle=False): +def get_minibatches_idx(n, minibatch_size, shuffle=False): """ Used to shuffle the dataset at each iteration. """ @@ -30,17 +30,16 @@ def get_minibatches_idx(n, nb_batches, shuffle=False): minibatches = [] minibatch_start = 0 - for i in range(nb_batches): - if i < n % nb_batches: - minibatch_size = n // nb_batches + 1 - else: - minibatch_size = n // nb_batches - + for i in range(n // minibatch_size): minibatches.append(idx_list[minibatch_start: minibatch_start + minibatch_size]) minibatch_start += minibatch_size - return zip(range(nb_batches), minibatches) + if (minibatch_start != n): + # Make a minibatch out of what is left + minibatches.append(idx_list[minibatch_start:]) + + return zip(range(len(minibatches)), minibatches) def get_dataset(name): @@ -446,11 +445,9 @@ def test_lstm( print 'Optimization' - kf_valid = get_minibatches_idx(len(valid[0]), - len(valid[0]) / valid_batch_size, + kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size, shuffle=True) - kf_test = get_minibatches_idx(len(test[0]), - len(test[0]) / valid_batch_size, + kf_test = get_minibatches_idx(len(test[0]), valid_batch_size, shuffle=True) history_errs = [] @@ -469,8 +466,7 @@ def test_lstm( n_samples = 0 # Get new shuffled index for the training set. - kf = get_minibatches_idx(len(train[0]), len(train[0])/batch_size, - shuffle=True) + kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True) for _, train_index in kf: uidx += 1 From 04c02d4f889a989a5db6ab51bea81de39d8baa65 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 13 Jan 2015 11:27:43 -0500 Subject: [PATCH 161/417] Fixed default dataset value in load_data() --- code/imdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/imdb.py b/code/imdb.py index c33884d6..085ab3f9 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -87,7 +87,7 @@ def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): # Load the dataset path = get_dataset_file( - path, "imdb.pkl.gz", + path, "imdb.pkl", "http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl") if path.endswith(".gz"): From 74b2e0c75e5a1be863c890510898ea42d2cf14fd Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 10:34:43 -0500 Subject: [PATCH 162/417] Filter for the max seq len when we load the dataset --- code/imdb.py | 23 ++++++++++++++++++++--- code/lstm.py | 32 +++++++++++++++++--------------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index 085ab3f9..c9d150e2 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -74,11 +74,19 @@ def get_dataset_file(dataset, default_dataset, origin): return dataset -def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): +def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1, maxlen=None): ''' Loads the dataset - :type dataset: string - :param dataset: the path to the dataset (here IMDB) + :type path: String + :param path: The path to the dataset (here IMDB) + :type n_words: int + :param n_words: The number of word to keep in the vocabulary. + All extra words are set to unknow (1). + :type valid_portion: float + :param valid_portion: The proportion of the full train set used for + the validation set. + :type maxlen: None or positive int + :param maxlen: the max sequence length we use in the train/valid set. ''' ############# @@ -98,6 +106,15 @@ def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1): train_set = cPickle.load(f) test_set = cPickle.load(f) f.close() + if maxlen: + new_train_set_x = [] + new_train_set_y = [] + for x, y in zip(train_set[0], train_set[1]): + if len(x) < maxlen: + new_train_set_x.append(x) + new_train_set_y.append(y) + train_set = (new_train_set_x, new_train_set_y) + del new_train_set_x, new_train_set_y # split training set into validation set train_set_x, train_set_y = train_set diff --git a/code/lstm.py b/code/lstm.py index 1946c0ad..995c91a8 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -373,22 +373,22 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): return valid_err -def test_lstm( +def train_lstm( dim_proj=128, # word embeding dimension and LSTM number of hidden units. - patience=10, # number of epoch to wait before early stop if no progress + patience=10, # Number of epoch to wait before early stop if no progress max_epochs=5000, # The maximum number of epoch to run - dispFreq=10, # display to stdout the training progress every N updates - decay_c=0., # weight decay for the classifier applied to the U weights. - lrate=0.0001, # learning rate for sgd (not used for adadelta and rmsprop) - n_words=10000, # vocabulary size - optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decay learning rate). + dispFreq=10, # Display to stdout the training progress every N updates + decay_c=0., # Weight decay for the classifier applied to the U weights. + lrate=0.0001, # Learning rate for sgd (not used for adadelta and rmsprop) + n_words=10000, # Vocabulary size + optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate). encoder='lstm', # TODO: can be removed must be lstm. saveto='lstm_model.npz', # The best model will be saved there - validFreq=10000, # after 1000 - saveFreq=100000, # save the parameters after every saveFreq updates - maxlen=100, # longer sequence get ignored - batch_size=64, # the batch size during training. - valid_batch_size=64, # The batch size during validation + validFreq=390, # Compute the validation error after this number of update. + saveFreq=1040, # Save the parameters after every saveFreq updates + maxlen=100, # Sequence longer then this get ignored + batch_size=16, # The batch size during training. + valid_batch_size=64, # The batch size used for validation/test set. dataset='imdb', # Parameter for extra option @@ -400,11 +400,13 @@ def test_lstm( # Model options model_options = locals().copy() + print "model options", model_options load_data, prepare_data = get_dataset(dataset) print 'Loading data' - train, valid, test = load_data(n_words=n_words, valid_portion=0.01) + train, valid, test = load_data(n_words=n_words, valid_portion=0.01, + maxlen=maxlen) ydim = numpy.max(train[1])+1 @@ -569,7 +571,7 @@ def test_lstm( theano.config.scan.allow_gc = False # See function train for all possible parameter and there definition. - test_lstm( + train_lstm( #reload_model="lstm_model.npz", - max_epochs=10, + max_epochs=100, ) From 43adeff4754cdd15340bb4cf8a7e53c88af58a9f Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 10:36:47 -0500 Subject: [PATCH 163/417] use an higher valid proportion, to make it move. --- code/lstm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 995c91a8..6762ef93 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -405,7 +405,7 @@ def train_lstm( load_data, prepare_data = get_dataset(dataset) print 'Loading data' - train, valid, test = load_data(n_words=n_words, valid_portion=0.01, + train, valid, test = load_data(n_words=n_words, valid_portion=0.05, maxlen=maxlen) ydim = numpy.max(train[1])+1 @@ -452,6 +452,9 @@ def train_lstm( kf_test = get_minibatches_idx(len(test[0]), valid_batch_size, shuffle=True) + print "%d train examples" % len(train[0]) + print "%d valid examples" % len(valid[0]) + print "%d test examples" % len(test[0]) history_errs = [] best_p = None bad_count = 0 From 2da912206d3d9987d3d2d48b0d647405273be6d4 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 10:43:40 -0500 Subject: [PATCH 164/417] catch ctrl-C --- code/lstm.py | 157 ++++++++++++++++++++++++++------------------------- 1 file changed, 81 insertions(+), 76 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 6762ef93..431c962f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -384,8 +384,8 @@ def train_lstm( optimizer=adadelta, # sgd, adadelta and rmsprop available, sgd very hard to use, not recommanded (probably need momentum and decaying learning rate). encoder='lstm', # TODO: can be removed must be lstm. saveto='lstm_model.npz', # The best model will be saved there - validFreq=390, # Compute the validation error after this number of update. - saveFreq=1040, # Save the parameters after every saveFreq updates + validFreq=370, # Compute the validation error after this number of update. + saveFreq=1110, # Save the parameters after every saveFreq updates maxlen=100, # Sequence longer then this get ignored batch_size=16, # The batch size during training. valid_batch_size=64, # The batch size used for validation/test set. @@ -467,80 +467,85 @@ def train_lstm( uidx = 0 # the number of update done estop = False # early stop start_time = time.clock() - for eidx in xrange(max_epochs): - n_samples = 0 - - # Get new shuffled index for the training set. - kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True) - - for _, train_index in kf: - uidx += 1 - use_noise.set_value(1.) - - # Select the random examples for this minibatch - y = [train[1][t] for t in train_index] - x = [train[0][t]for t in train_index] - - # Get the data in numpy.ndarray formet. - # It return something of the shape (minibatch maxlen, n samples) - x, mask, y = prepare_data(x, y, maxlen=maxlen) - if x is None: - print 'Minibatch with zero sample under length ', maxlen - continue - n_samples += x.shape[1] - - cost = f_grad_shared(x, mask, y) - f_update(lrate) - - if numpy.isnan(cost) or numpy.isinf(cost): - print 'NaN detected' - return 1., 1., 1. - - if numpy.mod(uidx, dispFreq) == 0: - print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost - - if numpy.mod(uidx, saveFreq) == 0: - print 'Saving...', - - if best_p is not None: - params = best_p - else: - params = unzip(tparams) - numpy.savez(saveto, history_errs=history_errs, **params) - pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) - print 'Done' - - if numpy.mod(uidx, validFreq) == 0: - use_noise.set_value(0.) - train_err = pred_error(f_pred, prepare_data, train, kf) - valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) - test_err = pred_error(f_pred, prepare_data, test, kf_test) - - history_errs.append([valid_err, test_err]) - - if (uidx == 0 or - valid_err <= numpy.array(history_errs)[:, - 0].min()): - - best_p = unzip(tparams) - bad_counter = 0 - - print ('Train ', train_err, 'Valid ', valid_err, - 'Test ', test_err) - - if (len(history_errs) > patience and - valid_err >= numpy.array(history_errs)[:-patience, - 0].min()): - bad_counter += 1 - if bad_counter > patience: - print 'Early Stop!' - estop = True - break - - print 'Seen %d samples' % n_samples - - if estop: - break + try: + for eidx in xrange(max_epochs): + n_samples = 0 + + # Get new shuffled index for the training set. + kf = get_minibatches_idx(len(train[0]), batch_size, shuffle=True) + + for _, train_index in kf: + uidx += 1 + use_noise.set_value(1.) + + # Select the random examples for this minibatch + y = [train[1][t] for t in train_index] + x = [train[0][t]for t in train_index] + + # Get the data in numpy.ndarray formet. + # It return something of the shape (minibatch maxlen, n samples) + x, mask, y = prepare_data(x, y, maxlen=maxlen) + if x is None: + print 'Minibatch with zero sample under length ', maxlen + continue + n_samples += x.shape[1] + + cost = f_grad_shared(x, mask, y) + f_update(lrate) + + if numpy.isnan(cost) or numpy.isinf(cost): + print 'NaN detected' + return 1., 1., 1. + + if numpy.mod(uidx, dispFreq) == 0: + print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost + + if numpy.mod(uidx, saveFreq) == 0: + print 'Saving...', + + if best_p is not None: + params = best_p + else: + params = unzip(tparams) + numpy.savez(saveto, history_errs=history_errs, **params) + pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) + print 'Done' + + if numpy.mod(uidx, validFreq) == 0: + use_noise.set_value(0.) + train_err = pred_error(f_pred, prepare_data, train, kf) + valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) + test_err = pred_error(f_pred, prepare_data, test, kf_test) + + history_errs.append([valid_err, test_err]) + + if (uidx == 0 or + valid_err <= numpy.array(history_errs)[:, + 0].min()): + + best_p = unzip(tparams) + bad_counter = 0 + + print ('Train ', train_err, 'Valid ', valid_err, + 'Test ', test_err) + + if (len(history_errs) > patience and + valid_err >= numpy.array(history_errs)[:-patience, + 0].min()): + bad_counter += 1 + if bad_counter > patience: + print 'Early Stop!' + estop = True + break + + print 'Seen %d samples' % n_samples + + if estop: + break + + except KeyboardInterrupt: + print "Training interupted" + end_time = time.clock() if best_p is not None: zipp(best_p, tparams) From 5482b180e327165168eb62a47c06e20c7d7425c4 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 13 Jan 2015 10:44:01 -0500 Subject: [PATCH 165/417] small clean up --- code/lstm.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 431c962f..00279ce0 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -559,11 +559,9 @@ def train_lstm( print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err - params = copy.copy(best_p) - numpy.savez(saveto, zipped_params=best_p, train_err=train_err, + numpy.savez(saveto, train_err=train_err, valid_err=valid_err, test_err=test_err, - history_errs=history_errs, **params) - + history_errs=history_errs, **best_p) print 'The code run for %d epochs, with %f sec/epochs' % ( (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) print >> sys.stderr, ('Training took %.1fs' % From c1756e8dd6f6d27408157d68158e4d6589b627c5 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Thu, 15 Jan 2015 14:55:04 -0500 Subject: [PATCH 166/417] Fix error in comment. --- code/convolutional_mlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 15b98a98..0d88240d 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -179,7 +179,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # Construct the second convolutional pooling layer # filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8) # maxpooling reduces this further to (8/2, 8/2) = (4, 4) - # 4D output tensor is thus of shape (nkerns[0], nkerns[1], 4, 4) + # 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4) layer1 = LeNetConvPoolLayer( rng, input=layer0.output, From 9018392ea487e37f6620cef6222bc3870f2a3dfb Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 14 Jan 2015 14:55:37 -0500 Subject: [PATCH 167/417] pep8 --- code/imdb.py | 2 -- code/lstm.py | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index c9d150e2..b8297482 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -4,8 +4,6 @@ import numpy -import theano - def prepare_data(seqs, labels, maxlen=None): """Create the matrices from the datasets. diff --git a/code/lstm.py b/code/lstm.py index 00279ce0..04b21e3c 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -2,7 +2,6 @@ Build a tweet sentiment analyzer ''' from collections import OrderedDict -import copy import cPickle as pkl import random import sys @@ -159,8 +158,8 @@ def lstm_layer(tparams, state_below, options, prefix='lstm', mask=None): def _slice(_x, n, dim): if _x.ndim == 3: - return _x[:, :, n*dim:(n+1)*dim] - return _x[:, n*dim:(n+1)*dim] + return _x[:, :, n * dim:(n + 1) * dim] + return _x[:, n * dim:(n + 1) * dim] def _step(m_, x_, h_, c_): preact = tensor.dot(h_, tparams[_p(prefix, 'U')]) @@ -243,7 +242,7 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) for rg2, g in zip(running_grads2, grads)] - f_grad_shared = theano.function([x, mask, y], cost, updates=zgup+rg2up, + f_grad_shared = theano.function([x, mask, y], cost, updates=zgup + rg2up, name='adadelta_f_grad_shared') updir = [-tensor.sqrt(ru2 + 1e-6) / tensor.sqrt(rg2 + 1e-6) * zg @@ -254,7 +253,7 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): for ru2, ud in zip(running_up2, updir)] param_up = [(p, p + ud) for p, ud in zip(tparams.values(), updir)] - f_update = theano.function([lr], [], updates=ru2up+param_up, + f_update = theano.function([lr], [], updates=ru2up + param_up, on_unused_input='ignore', name='adadelta_f_update') @@ -289,7 +288,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): running_grads2)] param_up = [(p, p + udn[1]) for p, udn in zip(tparams.values(), updir_new)] - f_update = theano.function([lr], [], updates=updir_new+param_up, + f_update = theano.function([lr], [], updates=updir_new + param_up, on_unused_input='ignore', name='rmsprop_f_update') @@ -321,7 +320,7 @@ def build_model(tparams, options): if options['use_dropout']: proj = dropout_layer(proj, use_noise, trng) - pred = tensor.nnet.softmax(tensor.dot(proj, tparams['U'])+tparams['b']) + pred = tensor.nnet.softmax(tensor.dot(proj, tparams['U']) + tparams['b']) f_pred_prob = theano.function([x, mask], pred, name='f_pred_prob') f_pred = theano.function([x, mask], pred.argmax(axis=1), name='f_pred') @@ -408,7 +407,7 @@ def train_lstm( train, valid, test = load_data(n_words=n_words, valid_portion=0.05, maxlen=maxlen) - ydim = numpy.max(train[1])+1 + ydim = numpy.max(train[1]) + 1 model_options['ydim'] = ydim @@ -432,7 +431,7 @@ def train_lstm( if decay_c > 0.: decay_c = theano.shared(numpy.float32(decay_c), name='decay_c') weight_decay = 0. - weight_decay += (tparams['U']**2).sum() + weight_decay += (tparams['U'] ** 2).sum() weight_decay *= decay_c cost += weight_decay @@ -460,9 +459,9 @@ def train_lstm( bad_count = 0 if validFreq == -1: - validFreq = len(train[0])/batch_size + validFreq = len(train[0]) / batch_size if saveFreq == -1: - saveFreq = len(train[0])/batch_size + saveFreq = len(train[0]) / batch_size uidx = 0 # the number of update done estop = False # early stop @@ -514,7 +513,8 @@ def train_lstm( if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) train_err = pred_error(f_pred, prepare_data, train, kf) - valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) + valid_err = pred_error(f_pred, prepare_data, valid, + kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) history_errs.append([valid_err, test_err]) From 906a30e37df0801a5d3c847affebe46453075ae7 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Thu, 15 Jan 2015 21:07:36 -0800 Subject: [PATCH 168/417] Small comment update --- code/imdb.py | 1 + code/lstm.py | 10 ++++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index b8297482..bc31a469 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -14,6 +14,7 @@ def prepare_data(seqs, labels, maxlen=None): if maxlen is set, we will cut all sequence to this maximum lenght. + This swap the axis! """ # x: a list of sentences lengths = [len(s) for s in seqs] diff --git a/code/lstm.py b/code/lstm.py index 04b21e3c..e1107e84 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -481,12 +481,10 @@ def train_lstm( y = [train[1][t] for t in train_index] x = [train[0][t]for t in train_index] - # Get the data in numpy.ndarray formet. - # It return something of the shape (minibatch maxlen, n samples) - x, mask, y = prepare_data(x, y, maxlen=maxlen) - if x is None: - print 'Minibatch with zero sample under length ', maxlen - continue + # Get the data in numpy.ndarray format + # This swap the axis! + # Return something of shape (minibatch maxlen, n samples) + x, mask, y = prepare_data(x, y) n_samples += x.shape[1] cost = f_grad_shared(x, mask, y) From 26914e4080661abf4211e6d129fbe01206a51334 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Thu, 15 Jan 2015 21:08:34 -0800 Subject: [PATCH 169/417] Add a parameter to make the test set smaller --- code/lstm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/lstm.py b/code/lstm.py index e1107e84..fd650de0 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -395,6 +395,7 @@ def train_lstm( use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. reload_model="", # Path to a saved model we want to start from. + test_size=-1, # If >0, we will trunc the test set to this number of example. ): # Model options @@ -406,6 +407,8 @@ def train_lstm( print 'Loading data' train, valid, test = load_data(n_words=n_words, valid_portion=0.05, maxlen=maxlen) + if test_size > 0: + test = (test[0][:test_size], test[1][:test_size]) ydim = numpy.max(train[1]) + 1 @@ -578,4 +581,5 @@ def train_lstm( train_lstm( #reload_model="lstm_model.npz", max_epochs=100, + test_size=500, ) From ec112faf5ba886a3e32127fedceef4da465d0c51 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 21 Jan 2015 09:33:13 -0800 Subject: [PATCH 170/417] use floatX for lstm --- code/imdb.py | 3 ++- code/lstm.py | 53 ++++++++++++++++++++++++++-------------------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index bc31a469..6c70cd70 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -3,6 +3,7 @@ import os import numpy +import theano def prepare_data(seqs, labels, maxlen=None): @@ -39,7 +40,7 @@ def prepare_data(seqs, labels, maxlen=None): maxlen = numpy.max(lengths) x = numpy.zeros((maxlen, n_samples)).astype('int64') - x_mask = numpy.zeros((maxlen, n_samples)).astype('float32') + x_mask = numpy.zeros((maxlen, n_samples)).astype(theano.config.floatX) for idx, s in enumerate(seqs): x[:lengths[idx], idx] = s x_mask[:lengths[idx], idx] = 1. diff --git a/code/lstm.py b/code/lstm.py index fd650de0..96accd70 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -9,6 +9,7 @@ import numpy import theano +from theano import config import theano.tensor as tensor from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams @@ -17,6 +18,10 @@ datasets = {'imdb': (imdb.load_data, imdb.prepare_data)} +def numpy_floatX(data): + return numpy.asarray(data, dtype=config.floatX) + + def get_minibatches_idx(n, minibatch_size, shuffle=False): """ Used to shuffle the dataset at each iteration. @@ -85,14 +90,14 @@ def init_params(options): # embedding randn = numpy.random.rand(options['n_words'], options['dim_proj']) - params['Wemb'] = (0.01 * randn).astype('float32') + params['Wemb'] = (0.01 * randn).astype(config.floatX) params = get_layer(options['encoder'])[0](options, params, prefix=options['encoder']) # classifier params['U'] = 0.01 * numpy.random.randn(options['dim_proj'], - options['ydim']).astype('float32') - params['b'] = numpy.zeros((options['ydim'],)).astype('float32') + options['ydim']).astype(config.floatX) + params['b'] = numpy.zeros((options['ydim'],)).astype(config.floatX) return params @@ -122,7 +127,7 @@ def get_layer(name): def ortho_weight(ndim): W = numpy.random.randn(ndim, ndim) u, s, v = numpy.linalg.svd(W) - return u.astype('float32') + return u.astype(config.floatX) def param_init_lstm(options, params, prefix='lstm'): @@ -142,7 +147,7 @@ def param_init_lstm(options, params, prefix='lstm'): ortho_weight(options['dim_proj'])], axis=1) params[_p(prefix, 'U')] = U b = numpy.zeros((4 * options['dim_proj'],)) - params[_p(prefix, 'b')] = b.astype('float32') + params[_p(prefix, 'b')] = b.astype(config.floatX) return params @@ -185,9 +190,11 @@ def _step(m_, x_, h_, c_): dim_proj = options['dim_proj'] rval, updates = theano.scan(_step, sequences=[mask, state_below], - outputs_info=[tensor.alloc(0., n_samples, + outputs_info=[tensor.alloc(numpy_floatX(0.), + n_samples, dim_proj), - tensor.alloc(0., n_samples, + tensor.alloc(numpy_floatX(0.), + n_samples, dim_proj)], name=_p(prefix, '_layers'), n_steps=nsteps) @@ -228,13 +235,13 @@ def sgd(lr, tparams, grads, x, mask, y, cost): def adadelta(lr, tparams, grads, x, mask, y, cost): - zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), + zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] - running_up2 = [theano.shared(p.get_value() * numpy.float32(0.), + running_up2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rup2' % k) for k, p in tparams.iteritems()] - running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), + running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] @@ -261,13 +268,13 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): def rmsprop(lr, tparams, grads, x, mask, y, cost): - zipped_grads = [theano.shared(p.get_value() * numpy.float32(0.), + zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] - running_grads = [theano.shared(p.get_value() * numpy.float32(0.), + running_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad' % k) for k, p in tparams.iteritems()] - running_grads2 = [theano.shared(p.get_value() * numpy.float32(0.), + running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) for k, p in tparams.iteritems()] @@ -280,7 +287,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): updates=zgup + rgup + rg2up, name='rmsprop_f_grad_shared') - updir = [theano.shared(p.get_value() * numpy.float32(0.), + updir = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_updir' % k) for k, p in tparams.iteritems()] updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) @@ -299,10 +306,10 @@ def build_model(tparams, options): trng = RandomStreams(1234) # Used for dropout. - use_noise = theano.shared(numpy.float32(0.)) + use_noise = theano.shared(numpy_floatX(0.)) x = tensor.matrix('x', dtype='int64') - mask = tensor.matrix('mask', dtype='float32') + mask = tensor.matrix('mask', dtype=config.floatX) y = tensor.vector('y', dtype='int64') n_timesteps = x.shape[0] @@ -335,7 +342,7 @@ def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): the probabilities of new examples. """ n_samples = len(data[0]) - probs = numpy.zeros((n_samples, 2)).astype('float32') + probs = numpy.zeros((n_samples, 2)).astype(config.floatX) n_done = 0 @@ -367,7 +374,7 @@ def pred_error(f_pred, prepare_data, data, iterator, verbose=False): preds = f_pred(x, mask) targets = numpy.array(data[1])[valid_index] valid_err += (preds == targets).sum() - valid_err = 1. - numpy.float32(valid_err) / len(data[0]) + valid_err = 1. - numpy_floatX(valid_err) / len(data[0]) return valid_err @@ -395,7 +402,7 @@ def train_lstm( use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. reload_model="", # Path to a saved model we want to start from. - test_size=-1, # If >0, we will trunc the test set to this number of example. + test_size=-1, # If >0, we keep only this number of test example. ): # Model options @@ -432,7 +439,7 @@ def train_lstm( y, f_pred_prob, f_pred, cost) = build_model(tparams, model_options) if decay_c > 0.: - decay_c = theano.shared(numpy.float32(decay_c), name='decay_c') + decay_c = theano.shared(numpy_floatX(decay_c), name='decay_c') weight_decay = 0. weight_decay += (tparams['U'] ** 2).sum() weight_decay *= decay_c @@ -571,12 +578,6 @@ def train_lstm( if __name__ == '__main__': - - # We must have floatX=float32 for this tutorial to work correctly. - theano.config.floatX = "float32" - # The next line is the new Theano default. This is a speed up. - theano.config.scan.allow_gc = False - # See function train for all possible parameter and there definition. train_lstm( #reload_model="lstm_model.npz", From 6a32f03503ef8128bc695705410fe8b4194d26fa Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 30 Jan 2015 21:53:12 -0500 Subject: [PATCH 171/417] sort dataset by length to speed up error computation --- code/imdb.py | 27 +++++++++++++++++++++++++-- code/lstm.py | 9 ++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/code/imdb.py b/code/imdb.py index 6c70cd70..21e0e376 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -74,8 +74,9 @@ def get_dataset_file(dataset, default_dataset, origin): return dataset -def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1, maxlen=None): - ''' Loads the dataset +def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1, maxlen=None, + sort_by_len=True): + '''Loads the dataset :type path: String :param path: The path to the dataset (here IMDB) @@ -87,6 +88,12 @@ def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1, maxlen=None): the validation set. :type maxlen: None or positive int :param maxlen: the max sequence length we use in the train/valid set. + :type sort_by_len: bool + :name sort_by_len: Sort by the sequence lenght for the train, + valid and test set. This allow faster execution as it cause + less padding per minibatch. Another mechanism must be used to + shuffle the train set at each epoch. + ''' ############# @@ -140,6 +147,22 @@ def remove_unk(x): valid_set_x = remove_unk(valid_set_x) test_set_x = remove_unk(test_set_x) + def len_argsort(seq): + return sorted(range(len(seq)), key=lambda x: len(seq[x])) + + if sort_by_len: + sorted_index = len_argsort(test_set_x) + test_set_x = [test_set_x[i] for i in sorted_index] + test_set_y = [test_set_y[i] for i in sorted_index] + + sorted_index = len_argsort(valid_set_x) + valid_set_x = [valid_set_x[i] for i in sorted_index] + valid_set_y = [valid_set_y[i] for i in sorted_index] + + sorted_index = len_argsort(train_set_x) + train_set_x = [train_set_x[i] for i in sorted_index] + train_set_y = [train_set_y[i] for i in sorted_index] + train = (train_set_x, train_set_y) valid = (valid_set_x, valid_set_y) test = (test_set_x, test_set_y) diff --git a/code/lstm.py b/code/lstm.py index 96accd70..1c67a446 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -456,10 +456,8 @@ def train_lstm( print 'Optimization' - kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size, - shuffle=True) - kf_test = get_minibatches_idx(len(test[0]), valid_batch_size, - shuffle=True) + kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size) + kf_test = get_minibatches_idx(len(test[0]), valid_batch_size) print "%d train examples" % len(train[0]) print "%d valid examples" % len(valid[0]) @@ -561,7 +559,8 @@ def train_lstm( best_p = unzip(tparams) use_noise.set_value(0.) - train_err = pred_error(f_pred, prepare_data, train, kf) + kf_train_sorted = get_minibatches_idx(len(train[0]), batch_size) + train_err = pred_error(f_pred, prepare_data, train, kf_train_sorted) valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) From 2f14fbfd98516eff1e99024769433f60d64d5803 Mon Sep 17 00:00:00 2001 From: Frederic Date: Sat, 31 Jan 2015 14:47:36 -0500 Subject: [PATCH 172/417] Try to repair travis failure to install our dependency --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ae499e46..5d0fa43f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ language: c # command to install dependencies before_install: #zlib1g-dev is needed to allow PIL to uncompress the dataset. + - sudo apt-get update - sudo apt-get install -qq libatlas3gf-base libatlas-dev zlib1g-dev zip unzip zlibc libzip-dev libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev python-numpy python-scipy python-pip python-nose python-yaml pyflakes python-imaging install: From 6e387832af57cc17f93e0583fec00ed04f7f700d Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 3 Feb 2015 09:39:03 -0500 Subject: [PATCH 173/417] Keep random subset of the test set --- code/lstm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 1c67a446..3e1c2525 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -415,7 +415,13 @@ def train_lstm( train, valid, test = load_data(n_words=n_words, valid_portion=0.05, maxlen=maxlen) if test_size > 0: - test = (test[0][:test_size], test[1][:test_size]) + # The test set is sorted by size, but we want to keep random + # size example. So we must select a random selection of the + # examples. + idx = numpy.arange(len(test[0])) + random.shuffle(idx) + idx = idx[:test_size] + test = ([test[0][n] for n in idx], [test[1][n] for n in idx]) ydim = numpy.max(train[1]) + 1 From c5696d0ea04efed47eed1fd92a6c8f138673977e Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Thu, 12 Feb 2015 09:44:35 -0500 Subject: [PATCH 174/417] use transpose instead of 2 swapaxes for clarity --- doc/lenet.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 8047cb28..a1791965 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -261,10 +261,11 @@ Let's have a little bit of fun with this... # open random image of dimensions 639x516 img = Image.open(open('doc/images/3wolfmoon.jpg')) + # dimensions are (height, width, channel) img = numpy.asarray(img, dtype='float64') / 256. # put image in 4D tensor of shape (1, 3, height, width) - img_ = img.swapaxes(0, 2).swapaxes(1, 2).reshape(1, 3, 639, 516) + img_ = img.transpose(2,0,1).reshape(1, 3, 639, 516) filtered_img = f(img_) # plot original image and first and second components of output From 205c493605a12ca08c797048a12721d3e6eb2a9c Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Thu, 12 Feb 2015 09:56:21 -0500 Subject: [PATCH 175/417] add missing import --- doc/lenet.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/lenet.txt b/doc/lenet.txt index a1791965..0abace23 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -194,7 +194,12 @@ one of Figure 1. The input consists of 3 features maps (an RGB color image) of s .. code-block:: python + import theano + from theano import tensor as T from theano.tensor.nnet import conv + + import numpy + rng = numpy.random.RandomState(23455) # instantiate 4D tensor for input From 8e91dc30d092e7a32f0580a1a142f10f24bc8c81 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 13 Feb 2015 14:46:45 -0500 Subject: [PATCH 176/417] pep8 --- doc/lenet.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 0abace23..ff210a6d 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -270,7 +270,7 @@ Let's have a little bit of fun with this... img = numpy.asarray(img, dtype='float64') / 256. # put image in 4D tensor of shape (1, 3, height, width) - img_ = img.transpose(2,0,1).reshape(1, 3, 639, 516) + img_ = img.transpose(2, 0, 1).reshape(1, 3, 639, 516) filtered_img = f(img_) # plot original image and first and second components of output From ff00edbcec16bec021c7b5f9f3bb4fc0a142d893 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 26 Mar 2015 16:13:17 -0400 Subject: [PATCH 177/417] Add test for lstm --- code/lstm.py | 10 +++++----- code/test.py | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 3e1c2525..14f78a02 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -511,7 +511,7 @@ def train_lstm( if numpy.mod(uidx, dispFreq) == 0: print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost - if numpy.mod(uidx, saveFreq) == 0: + if saveto and numpy.mod(uidx, saveFreq) == 0: print 'Saving...', if best_p is not None: @@ -571,10 +571,10 @@ def train_lstm( test_err = pred_error(f_pred, prepare_data, test, kf_test) print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err - - numpy.savez(saveto, train_err=train_err, - valid_err=valid_err, test_err=test_err, - history_errs=history_errs, **best_p) + if saveto: + numpy.savez(saveto, train_err=train_err, + valid_err=valid_err, test_err=test_err, + history_errs=history_errs, **best_p) print 'The code run for %d epochs, with %f sec/epochs' % ( (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) print >> sys.stderr, ('Training took %.1fs' % diff --git a/code/test.py b/code/test.py index df109332..18ea2a91 100644 --- a/code/test.py +++ b/code/test.py @@ -12,6 +12,7 @@ import rnnrbm import SdA import rnnslu +import lstm def test_rnnslu(): @@ -61,6 +62,10 @@ def test_rnnrbm(): rnnrbm.test_rnnrbm(num_epochs=1) +def test_lstm(): + lstm.train_lstm(max_epochs=1, test_size=1000, saveto='') + + def speed(): """ This fonction modify the configuration theano and don't restore it! From 5eeea378aef893a9e392f847f64634fe0e47d9e9 Mon Sep 17 00:00:00 2001 From: Frederic Date: Fri, 27 Mar 2015 10:26:46 -0400 Subject: [PATCH 178/417] Add lstm to the speed test --- code/test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 18ea2a91..543ad873 100644 --- a/code/test.py +++ b/code/test.py @@ -72,7 +72,7 @@ def speed(): """ algo = ['logistic_sgd', 'logistic_cg', 'mlp', 'convolutional_mlp', - 'dA', 'SdA', 'DBN', 'rbm', 'rnnrbm', 'rnnslu'] + 'dA', 'SdA', 'DBN', 'rbm', 'rnnrbm', 'rnnslu', 'lstm'] to_exec = [True] * len(algo) # to_exec = [False] * len(algo) # to_exec[-1] = True @@ -87,9 +87,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.8, 22.5, 76.1, 73.7, 116.4, - 346.9, 381.9, 558.1, 186.3, 50.8]) + 346.9, 381.9, 558.1, 186.3, 50.8, 113.6]) expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, - 191.2, 226.8, 432.8, 176.2, 36.9]) + 191.2, 226.8, 432.8, 176.2, 36.9, 78.0]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -110,7 +110,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 21.5, - 11.8, 47.9, 290.1, 255.4, 72.4]) + 11.8, 47.9, 290.1, 255.4, 72.4, 17.0]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) @@ -167,6 +167,8 @@ def do_tests(): # 60 is recommended 'savemodel': False} time_test(m, l, 9, rnnslu.main, param=s) + time_test(m, l, 10, lstm.train_lstm, max_epochs=1, test_size=1000, + saveto='') return numpy.asarray(l) #test in float64 in FAST_RUN mode on the cpu From 9e7bd40b5f16069ffecaf4ed25556e97ce92b397 Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 30 Mar 2015 10:07:03 -0400 Subject: [PATCH 179/417] update timming --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 543ad873..772589c1 100644 --- a/code/test.py +++ b/code/test.py @@ -109,7 +109,7 @@ def speed(): # 1.35324519 1.7356905 1.12937868] expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, - 5.8, 21.5, + 5.8, 20.5, 11.8, 47.9, 290.1, 255.4, 72.4, 17.0]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] From 4c293fe73f81edd687ff2a04b95a8de72f76d664 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Thu, 2 Apr 2015 11:00:44 -0400 Subject: [PATCH 180/417] fixed the typo --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index c9c2a3dd..73627c3d 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -190,7 +190,7 @@ can run the code by calling: .. code-block:: bash - THEANO_FLAGS="floatX=float32" python train_lstm.py + THEANO_FLAGS="floatX=float32" python lstm.py The script will automatically download the data and decompress it. From 5fea1bb295c28c576375159cfdb398deda987097 Mon Sep 17 00:00:00 2001 From: "Milind S. Pandit" Date: Tue, 7 Apr 2015 15:36:49 -0700 Subject: [PATCH 181/417] Edits based on feedback from @lamblin. --- doc/SdA.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/SdA.txt b/doc/SdA.txt index 4548bd4f..289a8b0a 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -53,7 +53,7 @@ are trained, we can train the :math:`k+1`-th layer because we can now compute the code or latent representation from the layer below. Once all layers are pre-trained, the network goes through a second stage -of training called **fine-tuning**, +of training called **fine-tuning**. Here we consider **supervised fine-tuning** where we want to minimize prediction error on a supervised task. For this, we first add a logistic regression layer on top of the network (more precisely on the output code of the @@ -66,15 +66,14 @@ training. (See the :ref:`mlp` for details on the multilayer perceptron.) This can be easily implemented in Theano, using the class defined previously for a denoising autoencoder. We can see the stacked denoising -autoencoder as having two facades: One is a list of -autoencoders. The other is an MLP. During pre-training we use the first facade, i.e., we treat our model +autoencoder as having two facades: a list of +autoencoders, and an MLP. During pre-training we use the first facade, i.e., we treat our model as a list of autoencoders, and train each autoencoder seperately. In the -second stage of training, we use the second facade. These two -facades are linked +second stage of training, we use the second facade. These two facades are linked because: -* by the parameters shared by the autoencoders and the sigmoid layers of the MLP, and +* the autoencoders and the sigmoid layers of the MLP share parameters, and -* by feeding the latent representations of intermediate layers of the MLP as input to the autoencoders. +* the latent representations computed by intermediate layers of the MLP are fed as input to the autoencoders. .. literalinclude:: ../code/SdA.py :start-after: start-snippet-1 @@ -83,8 +82,8 @@ facades are linked ``self.sigmoid_layers`` will store the sigmoid layers of the MLP facade, while ``self.dA_layers`` will store the denoising autoencoder associated with the layers of the MLP. -Next, we construct ``n_layers`` denoising autoencoders and ``n_layers`` sigmoid -layers, where ``n_layers`` is the depth of our model. We use the +Next, we construct ``n_layers`` sigmoid layers and ``n_layers`` denoising +autoencoders, where ``n_layers`` is the depth of our model. We use the ``HiddenLayer`` class introduced in :ref:`mlp`, with one modification: we replace the ``tanh`` non-linearity with the logistic function :math:`s(x) = \frac{1}{1+e^{-x}}`). From d38cfdffc8188d3be51b62831f6f09dfd22a901d Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Mon, 20 Apr 2015 12:36:35 +0100 Subject: [PATCH 182/417] small typo fix; and remove gender-specific phrase. --- doc/dA.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/dA.txt b/doc/dA.txt index cfdbf080..d447810d 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -126,7 +126,7 @@ signal: .. literalinclude:: ../code/dA.py :pyobject: dA.get_reconstructed_input -And using these function we can compute the cost and the updates of +And using these functions we can compute the cost and the updates of one stochastic gradient descent step : .. literalinclude:: ../code/dA.py @@ -466,7 +466,7 @@ since we neglect the biases and plot the weights up to a multiplicative constant (weights are converted to values between 0 and 1). To plot our filters we will need the help of ``tile_raster_images`` (see -:ref:`how-to-plot`) so we urge the reader to familiarize himself with it. Also +:ref:`how-to-plot`) so we urge the reader to study it. Also using the help of the Python Image Library, the following lines of code will save the filters as an image : From f9108d359a0c4ac0ffcdd967004c05bf98afe027 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Fri, 1 May 2015 19:06:56 -0400 Subject: [PATCH 183/417] Fix and deduplication in denoising autoencoder tutorial --- code/dA.py | 9 +- doc/dA.txt | 247 ++++------------------------------------------------- 2 files changed, 22 insertions(+), 234 deletions(-) diff --git a/code/dA.py b/code/dA.py index e1debf7a..19457aac 100644 --- a/code/dA.py +++ b/code/dA.py @@ -49,7 +49,6 @@ import Image -# start-snippet-1 class dA(object): """Denoising Auto-Encoder class (dA) @@ -191,7 +190,6 @@ def __init__( self.x = input self.params = [self.W, self.b, self.b_prime] - # end-snippet-1 def get_corrupted_input(self, input, corruption_level): """This function keeps ``1-corruption_level`` entries of the inputs the @@ -284,13 +282,16 @@ def test_dA(learning_rate=0.1, training_epochs=15, # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size + # start-snippet-2 # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images + # end-snippet-2 if not os.path.isdir(output_folder): os.makedirs(output_folder) os.chdir(output_folder) + #################################### # BUILDING THE MODEL NO CORRUPTION # #################################### @@ -348,6 +349,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, tile_spacing=(1, 1))) image.save('filters_corruption_0.png') + # start-snippet-3 ##################################### # BUILDING THE MODEL CORRUPTION 30% # ##################################### @@ -399,12 +401,15 @@ def test_dA(learning_rate=0.1, training_epochs=15, print >> sys.stderr, ('The 30% corruption code for file ' + os.path.split(__file__)[1] + ' ran for %.2fm' % (training_time / 60.)) + # end-snippet-3 + # start-snippet-4 image = Image.fromarray(tile_raster_images( X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), tile_spacing=(1, 1))) image.save('filters_corruption_30.png') + # end-snippet-4 os.chdir('../') diff --git a/doc/dA.txt b/doc/dA.txt index d447810d..8ff26354 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -109,8 +109,7 @@ using tied weights in this tutorial, :math:`\mathbf{W}^T` will be used for :math:`\mathbf{W'}`): .. literalinclude:: ../code/dA.py - :start-after: start-snippet-1 - :end-before: end-snippet-1 + :pyobject: dA.__init__ Note that we pass the symbolic ``input`` to the autoencoder as a parameter. This is so that we can concatenate layers of autoencoders to form a deep @@ -212,23 +211,8 @@ corruption mechanism of randomly masking entries of the input by making them zero. The code below does just that : -.. code-block:: python - - from theano.tensor.shared_randomstreams import RandomStreams - - def get_corrupted_input(self, input, corruption_level): - """ This function keeps ``1-corruption_level`` entries of the inputs the same - and zero-out randomly selected subset of size ``coruption_level`` - Note : first argument of theano.rng.binomial is the shape(size) of - random numbers that it should produce - second argument is the number of trials - third argument is the probability of success of any trial - - this will produce an array of 0s and 1s where 1 has a probability of - 1 - ``corruption_level`` and 0 with ``corruption_level`` - """ - return self.theano_rng.binomial(size=input.shape, n=1, p=1 - corruption_level) * input - +.. literalinclude:: ../code/dA.py + :pyobject: dA.get_corrupted_input In the stacked autoencoder class (:ref:`stacked_autoencoders`) the weights of @@ -239,172 +223,8 @@ new ones will be constructed. The final denoising autoencoder class becomes : -.. code-block:: python - - class dA(object): - """Denoising Auto-Encoder class (dA) - - A denoising autoencoders tries to reconstruct the input from a corrupted - version of it by projecting it first in a latent space and reprojecting - it afterwards back in the input space. Please refer to Vincent et al.,2008 - for more details. If x is the input then equation (1) computes a partially - destroyed version of x by means of a stochastic mapping q_D. Equation (2) - computes the projection of the input into the latent space. Equation (3) - computes the reconstruction of the input, while equation (4) computes the - reconstruction error. - - .. math:: - - \tilde{x} ~ q_D(\tilde{x}|x) (1) - - y = s(W \tilde{x} + b) (2) - - x = s(W' y + b') (3) - - L(x,z) = -sum_{k=1}^d [x_k \log z_k + (1-x_k) \log( 1-z_k)] (4) - - """ - - def __init__(self, numpy_rng, theano_rng=None, input=None, n_visible=784, n_hidden=500, - W=None, bhid=None, bvis=None): - """ - Initialize the dA class by specifying the number of visible units (the - dimension d of the input ), the number of hidden units ( the dimension - d' of the latent or hidden space ) and the corruption level. The - constructor also receives symbolic variables for the input, weights and - bias. Such a symbolic variables are useful when, for example the input is - the result of some computations, or when weights are shared between the - dA and an MLP layer. When dealing with SdAs this always happens, - the dA on layer 2 gets as input the output of the dA on layer 1, - and the weights of the dA are used in the second stage of training - to construct an MLP. - - :type numpy_rng: numpy.random.RandomState - :param numpy_rng: number random generator used to generate weights - - :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams - :param theano_rng: Theano random generator; if None is given one is generated - based on a seed drawn from `rng` - - :type input: theano.tensor.TensorType - :paran input: a symbolic description of the input or None for standalone - dA - - :type n_visible: int - :param n_visible: number of visible units - - :type n_hidden: int - :param n_hidden: number of hidden units - - :type W: theano.tensor.TensorType - :param W: Theano variable pointing to a set of weights that should be - shared belong the dA and another architecture; if dA should - be standalone set this to None - - :type bhid: theano.tensor.TensorType - :param bhid: Theano variable pointing to a set of biases values (for - hidden units) that should be shared belong dA and another - architecture; if dA should be standalone set this to None - - :type bvis: theano.tensor.TensorType - :param bvis: Theano variable pointing to a set of biases values (for - visible units) that should be shared belong dA and another - architecture; if dA should be standalone set this to None - - - """ - self.n_visible = n_visible - self.n_hidden = n_hidden - - # create a Theano random generator that gives symbolic random values - if not theano_rng : - theano_rng = RandomStreams(rng.randint(2 ** 30)) - - # note : W' was written as `W_prime` and b' as `b_prime` - if not W: - # W is initialized with `initial_W` which is uniformely sampled - # from -4.*sqrt(6./(n_visible+n_hidden)) and 4.*sqrt(6./(n_hidden+n_visible)) - # the output of uniform if converted using asarray to dtype - # theano.config.floatX so that the code is runable on GPU - initial_W = numpy.asarray(numpy_rng.uniform( - low=-4 * numpy.sqrt(6. / (n_hidden + n_visible)), - high=4 * numpy.sqrt(6. / (n_hidden + n_visible)), - size=(n_visible, n_hidden)), dtype=theano.config.floatX) - W = theano.shared(value=initial_W, name='W') - - if not bvis: - bvis = theano.shared(value = numpy.zeros(n_visible, - dtype=theano.config.floatX), name='bvis') - - if not bhid: - bhid = theano.shared(value=numpy.zeros(n_hidden, - dtype=theano.config.floatX), name='bhid') - - self.W = W - # b corresponds to the bias of the hidden - self.b = bhid - # b_prime corresponds to the bias of the visible - self.b_prime = bvis - # tied weights, therefore W_prime is W transpose - self.W_prime = self.W.T - self.theano_rng = theano_rng - # if no input is given, generate a variable representing the input - if input == None: - # we use a matrix because we expect a minibatch of several examples, - # each example being a row - self.x = T.dmatrix(name='input') - else: - self.x = input - - self.params = [self.W, self.b, self.b_prime] - - def get_corrupted_input(self, input, corruption_level): - """ This function keeps ``1-corruption_level`` entries of the inputs the same - and zero-out randomly selected subset of size ``coruption_level`` - Note : first argument of theano.rng.binomial is the shape(size) of - random numbers that it should produce - second argument is the number of trials - third argument is the probability of success of any trial - - this will produce an array of 0s and 1s where 1 has a probability of - 1 - ``corruption_level`` and 0 with ``corruption_level`` - """ - return self.theano_rng.binomial(size=input.shape, n=1, p=1 - corruption_level) * input - - - def get_hidden_values(self, input): - """ Computes the values of the hidden layer """ - return T.nnet.sigmoid(T.dot(input, self.W) + self.b) - - def get_reconstructed_input(self, hidden ): - """ Computes the reconstructed input given the values of the hidden layer """ - return T.nnet.sigmoid(T.dot(hidden, self.W_prime) + self.b_prime) - - def get_cost_updates(self, corruption_level, learning_rate): - """ This function computes the cost and the updates for one trainng - step of the dA """ - - tilde_x = self.get_corrupted_input(self.x, corruption_level) - y = self.get_hidden_values( tilde_x) - z = self.get_reconstructed_input(y) - # note : we sum over the size of a datapoint; if we are using minibatches, - # L will be a vector, with one entry per example in minibatch - L = -T.sum(self.x * T.log(z) + (1 - self.x) * T.log(1 - z), axis=1 ) - # note : L is now a vector, where each element is the cross-entropy cost - # of the reconstruction of the corresponding example of the - # minibatch. We need to compute the average of all these to get - # the cost of the minibatch - cost = T.mean(L) - - # compute the gradients of the cost of the `dA` with respect - # to its parameters - gparams = T.grad(cost, self.params) - # generate the list of updates - updates = [] - for param, gparam in zip(self.params, gparams): - updates.append((param, param - learning_rate * gparam)) - - return (cost, updates) +.. literalinclude:: ../code/dA.py + :pyobject: dA @@ -415,49 +235,15 @@ Putting it All Together It is easy now to construct an instance of our ``dA`` class and train it. -.. code-block:: python - - # allocate symbolic variables for the data - index = T.lscalar() # index to a [mini]batch - x = T.matrix('x') # the data is presented as rasterized images - - ###################### - # BUILDING THE MODEL # - ###################### - - rng = numpy.random.RandomState(123) - theano_rng = RandomStreams(rng.randint(2 ** 30)) - - da = dA(numpy_rng=rng, theano_rng=theano_rng, input=x, - n_visible=28 * 28, n_hidden=500) - - cost, updates = da.get_cost_updates(corruption_level=0.2, - learning_rate=learning_rate) - - - train_da = theano.function([index], cost, updates=updates, - givens = {x: train_set_x[index * batch_size: (index + 1) * batch_size]}) - - start_time = time.clock() - - ############ - # TRAINING # - ############ - - # go through training epochs - for epoch in xrange(training_epochs): - # go through trainng set - c = [] - for batch_index in xrange(n_train_batches): - c.append(train_da(batch_index)) - - print 'Training epoch %d, cost ' % epoch, numpy.mean(c) - - end_time = time.clock +.. literalinclude:: ../code/dA.py + :language: python + :start-after: start-snippet-2 + :end-before: end-snippet-2 - training_time = (end_time - start_time) +.. literalinclude:: ../code/dA.py + :start-after: start-snippet-3 + :end-before: end-snippet-3 - print ('Training took %f minutes' % (pretraining_time / 60.)) In order to get a feeling of what the network learned we are going to plot the filters (defined by the weight matrix). Bear in mind, however, @@ -470,12 +256,9 @@ To plot our filters we will need the help of ``tile_raster_images`` (see using the help of the Python Image Library, the following lines of code will save the filters as an image : -.. code-block:: python - - image = Image.fromarray(tile_raster_images(X=da.W.get_value(borrow=True).T, - img_shape=(28, 28), tile_shape=(10, 10), - tile_spacing=(1, 1))) - image.save('filters_corruption_30.png') +.. literalinclude:: ../code/dA.py + :start-after: start-snippet-4 + :end-before: end-snippet-4 Running the Code From 423f54d2daa67c02196737f17fc4365df6de76a1 Mon Sep 17 00:00:00 2001 From: Caglar Date: Mon, 11 May 2015 10:21:32 -0400 Subject: [PATCH 184/417] fixed the random number generators. --- code/lstm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 14f78a02..09eafe2a 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -17,6 +17,10 @@ datasets = {'imdb': (imdb.load_data, imdb.prepare_data)} +# Set the random number generators' seeds for consistency +SEED = 123 +random.seed(SEED) +numpy.random.seed(SEED) def numpy_floatX(data): return numpy.asarray(data, dtype=config.floatX) @@ -303,7 +307,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): def build_model(tparams, options): - trng = RandomStreams(1234) + trng = RandomStreams(SEED) # Used for dropout. use_noise = theano.shared(numpy_floatX(0.)) @@ -401,7 +405,7 @@ def train_lstm( noise_std=0., use_dropout=True, # if False slightly faster, but worst test error # This frequently need a bigger model. - reload_model="", # Path to a saved model we want to start from. + reload_model=None, # Path to a saved model we want to start from. test_size=-1, # If >0, we keep only this number of test example. ): From 29b167a328a8551f2f888813cd5da4e63d9d4390 Mon Sep 17 00:00:00 2001 From: Caglar Date: Mon, 11 May 2015 11:06:01 -0400 Subject: [PATCH 185/417] converted the code to use numpy random number generator. --- code/lstm.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 09eafe2a..4423840a 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -3,7 +3,6 @@ ''' from collections import OrderedDict import cPickle as pkl -import random import sys import time @@ -19,7 +18,6 @@ # Set the random number generators' seeds for consistency SEED = 123 -random.seed(SEED) numpy.random.seed(SEED) def numpy_floatX(data): @@ -34,7 +32,7 @@ def get_minibatches_idx(n, minibatch_size, shuffle=False): idx_list = numpy.arange(n, dtype="int32") if shuffle: - random.shuffle(idx_list) + numpy.random.shuffle(idx_list) minibatches = [] minibatch_start = 0 @@ -423,7 +421,7 @@ def train_lstm( # size example. So we must select a random selection of the # examples. idx = numpy.arange(len(test[0])) - random.shuffle(idx) + numpy.random.shuffle(idx) idx = idx[:test_size] test = ([test[0][n] for n in idx], [test[1][n] for n in idx]) @@ -472,6 +470,7 @@ def train_lstm( print "%d train examples" % len(train[0]) print "%d valid examples" % len(valid[0]) print "%d test examples" % len(test[0]) + history_errs = [] best_p = None bad_count = 0 @@ -589,7 +588,6 @@ def train_lstm( if __name__ == '__main__': # See function train for all possible parameter and there definition. train_lstm( - #reload_model="lstm_model.npz", max_epochs=100, test_size=500, ) From a6da8e39ee2e5061b79b5a7092837d767f6c6613 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 13 May 2015 09:51:14 -0400 Subject: [PATCH 186/417] Typo is comment --- code/mlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/mlp.py b/code/mlp.py index 6701378c..e4b95ea8 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -116,7 +116,7 @@ class MLP(object): that has one layer or more of hidden units and nonlinear activations. Intermediate layers usually have as activation function tanh or the sigmoid function (defined here by a ``HiddenLayer`` class) while the - top layer is a softamx layer (defined here by a ``LogisticRegression`` + top layer is a softmax layer (defined here by a ``LogisticRegression`` class). """ From 8222d24d1239b7207035bf8e3f81811a882fccec Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Wed, 13 May 2015 18:14:07 -0400 Subject: [PATCH 187/417] remove redundant bias addition --- code/lstm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 4423840a..8765158c 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -171,7 +171,6 @@ def _slice(_x, n, dim): def _step(m_, x_, h_, c_): preact = tensor.dot(h_, tparams[_p(prefix, 'U')]) preact += x_ - preact += tparams[_p(prefix, 'b')] i = tensor.nnet.sigmoid(_slice(preact, 0, options['dim_proj'])) f = tensor.nnet.sigmoid(_slice(preact, 1, options['dim_proj'])) From 5b625b5469975e96770762d92290529e74b47421 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 21 May 2015 10:13:31 -0400 Subject: [PATCH 188/417] Update timming following scan speed up --- code/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 772589c1..39d48f3b 100644 --- a/code/test.py +++ b/code/test.py @@ -87,9 +87,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.8, 22.5, 76.1, 73.7, 116.4, - 346.9, 381.9, 558.1, 186.3, 50.8, 113.6]) + 346.9, 381.9, 558.1, 130.4, 50.8, 113.6]) expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, - 191.2, 226.8, 432.8, 176.2, 36.9, 78.0]) + 191.2, 226.8, 432.8, 119.5, 36.9, 78.0]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -109,8 +109,8 @@ def speed(): # 1.35324519 1.7356905 1.12937868] expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, - 5.8, 20.5, - 11.8, 47.9, 290.1, 255.4, 72.4, 17.0]) + 5.8, 20.0, + 11.8, 47.9, 280.1, 132.8, 38.8, 10.5]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 777c75a9f514d2d5816df5cd355cc292fdb586f2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 21 May 2015 11:06:18 -0400 Subject: [PATCH 189/417] Make output more verbose to help work around no output limit of 10m --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5d0fa43f..3e1a3905 100644 --- a/.travis.yml +++ b/.travis.yml @@ -83,5 +83,5 @@ script: - ls - export THEANO_FLAGS=warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise - python --version - - nosetests $PART + - nosetests -v $PART From cfba0cd11bab6a563fbd34ee94ef598babe7faed Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 21 May 2015 14:55:44 -0400 Subject: [PATCH 190/417] Use bigger epsilon for float16 so that it does not become 0. --- code/lstm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/lstm.py b/code/lstm.py index 3e1c2525..1e2f7d21 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -332,7 +332,11 @@ def build_model(tparams, options): f_pred_prob = theano.function([x, mask], pred, name='f_pred_prob') f_pred = theano.function([x, mask], pred.argmax(axis=1), name='f_pred') - cost = -tensor.log(pred[tensor.arange(n_samples), y] + 1e-8).mean() + off = 1e-8 + if pred.dtype == 'float16': + off = 1e-6 + + cost = -tensor.log(pred[tensor.arange(n_samples), y] + off).mean() return use_noise, x, mask, y, f_pred_prob, f_pred, cost From 1caef8fd13294e7c750f1654fd766f1e79f3102e Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 09:31:54 -0400 Subject: [PATCH 191/417] replace rng with MRG --- code/DBN.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 8f9715f2..b0cc0569 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -8,7 +8,7 @@ import theano import theano.tensor as T -from theano.tensor.shared_randomstreams import RandomStreams +from theano.sandbox.rng_mrg import MRG_RandomStreams from logistic_sgd import LogisticRegression, load_data from mlp import HiddenLayer @@ -58,7 +58,7 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, assert self.n_layers > 0 if not theano_rng: - theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) + theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30)) # allocate symbolic variables for the data self.x = T.matrix('x') # the data is presented as rasterized images From 5cbd7b009c37957d852f3b900a8eee2f4b4ee159 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 09:48:33 -0400 Subject: [PATCH 192/417] Document adadelta and RMSPROP for lstm --- code/lstm.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/code/lstm.py b/code/lstm.py index 8c5bd2cd..0c69e69a 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -236,6 +236,34 @@ def sgd(lr, tparams, grads, x, mask, y, cost): def adadelta(lr, tparams, grads, x, mask, y, cost): + """ + An adaptive learning rate optimizer + + Parameters + ---------- + lr : float + Initial learning rate + tpramas: Theano SharedVariable + Model parameters + grads: Theano variable + Gradients of cost w.r.t to parameres + x: Theano variable + Model inputs + mask: Theano variable + Sequence mask + y: Theano variable + Targets + cost: Theano variable + Objective fucntion to minimize + + Notes + ----- + For more information, see [ADADELTA]_. + + .. [ADADELTA] Matthew D. Zeiler, *ADADELTA: An Adaptive Learning + Rate Method*, arXiv:1212.5701. + """ + zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] @@ -269,6 +297,36 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): def rmsprop(lr, tparams, grads, x, mask, y, cost): + """ + A variant of SGD that scales the step size by running average of the + recent step norms. + + Parameters + ---------- + lr : float + Initial learning rate + tpramas: Theano SharedVariable + Model parameters + grads: Theano variable + Gradients of cost w.r.t to parameres + x: Theano variable + Model inputs + mask: Theano variable + Sequence mask + y: Theano variable + Targets + cost: Theano variable + Objective fucntion to minimize + + Notes + ----- + For more information, see [Hint2014]_. + + .. [Hint2014] Geoff Hinton, *Neural Networks for Machine Learning*, + lecture 6a, + http://cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf + """ + zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) for k, p in tparams.iteritems()] From 885e6d40840a06b8e24c8295c73fcb2194811de4 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 10:01:51 -0400 Subject: [PATCH 193/417] Clarify difference between modern conv2d and original LeNet --- doc/lenet.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/lenet.txt b/doc/lenet.txt index ff210a6d..c8ed08e5 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -400,6 +400,14 @@ tensors. These are then flattened to a 2D matrix of rasterized feature maps, to be compatible with our previous MLP implementation. +.. note:: + Please note that the convolution operation in majority of of the recent + models and almost all the library codes available is slightly different + than the original LeNet model. In modern conv2d, for each feature map + The result sums all the convolved input feature maps with the corresponding + kernel. As opposed to some of the earlier works that used to select only + some of the input feature maps. + Putting it All Together +++++++++++++++++++++++ From 8f54f6d5e6974c35178d117cf5e22c350552a588 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 10:51:48 -0400 Subject: [PATCH 194/417] change lr dtype --- code/lstm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 0c69e69a..cc4ab748 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -241,7 +241,7 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): Parameters ---------- - lr : float + lr : Theano SharedVariable Initial learning rate tpramas: Theano SharedVariable Model parameters @@ -303,7 +303,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): Parameters ---------- - lr : float + lr : Theano SharedVariable Initial learning rate tpramas: Theano SharedVariable Model parameters From 8f002a506595c9b0bacabc35b48dc085acedbc91 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 10:50:37 -0400 Subject: [PATCH 195/417] add predict option to logistic regression mend mend --- code/logistic_sgd.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 599f5658..feccf9a9 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -44,6 +44,7 @@ import theano import theano.tensor as T +from theano.gof import graph class LogisticRegression(object): @@ -415,6 +416,10 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, ) ) + # save the best model + with open('best_model.pkl', 'w') as f: + cPickle.dump(classifier, f) + if patience <= iter: done_looping = True break @@ -433,5 +438,37 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, os.path.split(__file__)[1] + ' ran for %.1fs' % ((end_time - start_time))) + +def predict(): + """ + An example of how to load a train model and use it + to predict labels. + """ + + # load the saved model + classifier = cPickle.load(open('best_model.pkl')) + y_pred = classifier.y_pred + + # find the input to theano graph + inputs = graph.inputs([y_pred]) + # select only x + inputs = [item for item in inputs if item.name == 'x'] + + # compile a predictor function + predict_model = theano.function( + inputs=inputs, + outputs=y_pred) + + # We can test it on some examples from test test + dataset='mnist.pkl.gz' + datasets = load_data(dataset) + test_set_x, test_set_y = datasets[2] + test_set_x = test_set_x.get_value() + + predicted_values = predict_model(test_set_x[:10]) + print ("Predicted values for the first 10 examples in test set:") + print predicted_values + + if __name__ == '__main__': sgd_optimization_mnist() From fc762a7feb58c53262245fc9ee5df856a91970b6 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Mon, 22 Jun 2015 14:50:30 -0400 Subject: [PATCH 196/417] update docs --- code/logistic_sgd.py | 2 +- doc/logreg.txt | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index feccf9a9..85236ecb 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -441,7 +441,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, def predict(): """ - An example of how to load a train model and use it + An example of how to load a trained model and use it to predict labels. """ diff --git a/doc/logreg.txt b/doc/logreg.txt index ab27f95b..1b851bba 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -264,6 +264,20 @@ approximately 1.936 epochs/sec and it took 75 epochs to reach a test error of 7.489%. On the GPU the code does almost 10.0 epochs/sec. For this instance we used a batch size of 600. + +Prediction Using a Trained Model ++++++++++++++++++++++++++++++++ + +``sgd_optimization_mnist`` serialize and pickle the model each time new +lowest validation error is reached. We can reload this model and predict +labels of new data. ``predict`` function shows an example of how +this could be done. + +.. literalinclude:: ../code/logistic_sgd.py + :start-after: ' ran for %.1fs' % ((end_time - start_time))) + :end-before: if __name__ == '__main__': + + .. rubric:: Footnotes .. [#f1] For smaller datasets and simpler models, more sophisticated descent From 75ff7e837b548e6b25c3be05370d665a6bb8b268 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Wed, 24 Jun 2015 20:20:56 -0400 Subject: [PATCH 197/417] Explicitly install "six" since theano is installed with "--no-deps" --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3e1a3905..af5d82ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ install: # - "pip install -q Pillow --use-mirrors" #If we don't install numpy before SciPy 0.10.1, the SciPy installations fails. # - "pip install -q scipy --use-mirrors" + - "sudo pip install six" - "sudo pip install --no-deps git+git://github.com/Theano/Theano.git" env: From 68c8e4eaed44c9975931984894391e3854f3312d Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 25 Jun 2015 13:45:31 -0400 Subject: [PATCH 198/417] Stop using --no-deps when installing theano. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index af5d82ac..08536413 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,7 @@ install: # - "pip install -q Pillow --use-mirrors" #If we don't install numpy before SciPy 0.10.1, the SciPy installations fails. # - "pip install -q scipy --use-mirrors" - - "sudo pip install six" - - "sudo pip install --no-deps git+git://github.com/Theano/Theano.git" + - "sudo pip install git+git://github.com/Theano/Theano.git" env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" From 1245f57cdf29e661547739ce8e4318b06e59d6d9 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 25 Jun 2015 14:56:20 -0400 Subject: [PATCH 199/417] Use miniconda to get supported versions of dependecies for Theano. --- .travis.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08536413..7873dedf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,22 @@ # After changing this file, check it on: # http://lint.travis-ci.org/ -#We can't get scipy installed with the python language -#So we will use the system python from the c language. -language: c -#language: python +language: python #python: -# - "2.5" # - "2.7" # - "3.2" # command to install dependencies before_install: -#zlib1g-dev is needed to allow PIL to uncompress the dataset. - - sudo apt-get update - - sudo apt-get install -qq libatlas3gf-base libatlas-dev zlib1g-dev zip unzip zlibc libzip-dev libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev python-numpy python-scipy python-pip python-nose python-yaml pyflakes python-imaging + - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh + - chmod +x miniconda.sh + - ./miniconda.sh -b + - export PATH=/home/travis/miniconda/bin:$PATH + - conda update --yes conda install: -# - "pip install -q numpy --use-mirrors" -# Use Pillow instead of PIL as it is better packaged -# - "pip install -q Pillow --use-mirrors" -#If we don't install numpy before SciPy 0.10.1, the SciPy installations fails. -# - "pip install -q scipy --use-mirrors" - - "sudo pip install git+git://github.com/Theano/Theano.git" + - conda create --yes -q -n pyenv mkl python=2.7 numpy scipy pip nose yaml pyflakes pillow pyparsing=1.5 + - source activate pyenv + - pip install git+git://github.com/Theano/Theano.git env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" From 5a719b5bbcf85b1955d2ae054d51c460ebb89d30 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Fri, 26 Jun 2015 11:18:52 -0400 Subject: [PATCH 200/417] updated the text --- doc/lenet.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index c8ed08e5..368b099b 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -401,12 +401,12 @@ to be compatible with our previous MLP implementation. .. note:: - Please note that the convolution operation in majority of of the recent - models and almost all the library codes available is slightly different - than the original LeNet model. In modern conv2d, for each feature map - The result sums all the convolved input feature maps with the corresponding - kernel. As opposed to some of the earlier works that used to select only - some of the input feature maps. + Note that the term convolution could corresponds to different mathematical operations. + 1. theano.tensor.nnet.conv2d which is the most common one in almost all of the recent published convolutional models. In this op for each output feature map, all the input feature maps are summed together after being convolved with the filter. + 2. Original LeNet model: In this work for each output feature map, only subset of input feature maps were selected. + 3. The convolution used in signal processing: theano.tensor.signal.conv.conv2d + + Putting it All Together +++++++++++++++++++++++ From 373dfbfea523bb776e77855c7ef87ad6fe1faaff Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Thu, 25 Jun 2015 20:13:19 -0400 Subject: [PATCH 201/417] add inputs the classifer --- code/logistic_sgd.py | 14 +++++--------- doc/logreg.txt | 3 +-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 85236ecb..33e9b39d 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -44,7 +44,6 @@ import theano import theano.tensor as T -from theano.gof import graph class LogisticRegression(object): @@ -110,6 +109,9 @@ def __init__(self, input, n_in, n_out): # parameters of the model self.params = [self.W, self.b] + # keep track of model input + self.input = input + def negative_log_likelihood(self, y): """Return the mean of the negative log-likelihood of the prediction of this model under a given target distribution. @@ -447,17 +449,11 @@ def predict(): # load the saved model classifier = cPickle.load(open('best_model.pkl')) - y_pred = classifier.y_pred - - # find the input to theano graph - inputs = graph.inputs([y_pred]) - # select only x - inputs = [item for item in inputs if item.name == 'x'] # compile a predictor function predict_model = theano.function( - inputs=inputs, - outputs=y_pred) + inputs=[classifier.input], + outputs=classifier.y_pred) # We can test it on some examples from test test dataset='mnist.pkl.gz' diff --git a/doc/logreg.txt b/doc/logreg.txt index 1b851bba..57065e99 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -274,8 +274,7 @@ labels of new data. ``predict`` function shows an example of how this could be done. .. literalinclude:: ../code/logistic_sgd.py - :start-after: ' ran for %.1fs' % ((end_time - start_time))) - :end-before: if __name__ == '__main__': + :pyobject: predict .. rubric:: Footnotes From a8bde076f20f75fc01a17a4eba9dc3f954339935 Mon Sep 17 00:00:00 2001 From: Changxu Date: Sun, 28 Jun 2015 23:43:43 +0800 Subject: [PATCH 202/417] replace time.clock() by time.time() --- code/DBN.py | 8 ++++---- code/SdA.py | 8 ++++---- code/cA.py | 4 ++-- code/convolutional_mlp.py | 4 ++-- code/dA.py | 8 ++++---- code/logistic_cg.py | 4 ++-- code/logistic_sgd.py | 4 ++-- code/lstm.py | 4 ++-- code/mlp.py | 4 ++-- code/rbm.py | 8 ++++---- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index b0cc0569..a18c00af 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -327,7 +327,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, k=k) print '... pre-training the model' - start_time = time.clock() + start_time = time.time() ## Pre-train layer-wise for i in xrange(dbn.n_layers): # go through pretraining epochs @@ -340,7 +340,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) - end_time = time.clock() + end_time = time.time() # end-snippet-2 print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + @@ -372,7 +372,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, best_validation_loss = numpy.inf test_score = 0. - start_time = time.clock() + start_time = time.time() done_looping = False epoch = 0 @@ -424,7 +424,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, done_looping = True break - end_time = time.clock() + end_time = time.time() print( ( 'Optimization complete with best validation score of %f %%, ' diff --git a/code/SdA.py b/code/SdA.py index fafa73b5..95cc971a 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -379,7 +379,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, batch_size=batch_size) print '... pre-training the model' - start_time = time.clock() + start_time = time.time() ## Pre-train layer-wise corruption_levels = [.1, .2, .3] for i in xrange(sda.n_layers): @@ -394,7 +394,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) - end_time = time.clock() + end_time = time.time() print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + @@ -427,7 +427,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, best_validation_loss = numpy.inf test_score = 0. - start_time = time.clock() + start_time = time.time() done_looping = False epoch = 0 @@ -471,7 +471,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, done_looping = True break - end_time = time.clock() + end_time = time.time() print( ( 'Optimization complete with best validation score of %f %%, ' diff --git a/code/cA.py b/code/cA.py index c7ccd2b0..209cc7fa 100644 --- a/code/cA.py +++ b/code/cA.py @@ -276,7 +276,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, } ) - start_time = time.clock() + start_time = time.time() ############ # TRAINING # @@ -293,7 +293,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, print 'Training epoch %d, reconstruction cost ' % epoch, numpy.mean( c_array[0]), ' jacobian norm ', numpy.mean(numpy.sqrt(c_array[1])) - end_time = time.clock() + end_time = time.time() training_time = (end_time - start_time) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 0d88240d..d4e996ea 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -274,7 +274,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, best_validation_loss = numpy.inf best_iter = 0 test_score = 0. - start_time = time.clock() + start_time = time.time() epoch = 0 done_looping = False @@ -326,7 +326,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, done_looping = True break - end_time = time.clock() + end_time = time.time() print('Optimization complete.') print('Best validation score of %f %% obtained at iteration %i, ' 'with test performance %f %%' % diff --git a/code/dA.py b/code/dA.py index 19457aac..08ace3c3 100644 --- a/code/dA.py +++ b/code/dA.py @@ -321,7 +321,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, } ) - start_time = time.clock() + start_time = time.time() ############ # TRAINING # @@ -336,7 +336,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost ' % epoch, numpy.mean(c) - end_time = time.clock() + end_time = time.time() training_time = (end_time - start_time) @@ -379,7 +379,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, } ) - start_time = time.clock() + start_time = time.time() ############ # TRAINING # @@ -394,7 +394,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost ' % epoch, numpy.mean(c) - end_time = time.clock() + end_time = time.time() training_time = (end_time - start_time) diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 05f562a1..429b523a 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -275,7 +275,7 @@ def callback(theta_value): # using scipy conjugate gradient optimizer import scipy.optimize print ("Optimizing using scipy.optimize.fmin_cg...") - start_time = time.clock() + start_time = time.time() best_w_b = scipy.optimize.fmin_cg( f=train_fn, x0=numpy.zeros((n_in + 1) * n_out, dtype=x.dtype), @@ -284,7 +284,7 @@ def callback(theta_value): disp=0, maxiter=n_epochs ) - end_time = time.clock() + end_time = time.time() print( ( 'Optimization complete with best validation score of %f %%, with ' diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 599f5658..1f521fe6 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -360,7 +360,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, best_validation_loss = numpy.inf test_score = 0. - start_time = time.clock() + start_time = time.time() done_looping = False epoch = 0 @@ -419,7 +419,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, done_looping = True break - end_time = time.clock() + end_time = time.time() print( ( 'Optimization complete with best validation score of %f %%,' diff --git a/code/lstm.py b/code/lstm.py index cc4ab748..b64970fb 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -543,7 +543,7 @@ def train_lstm( uidx = 0 # the number of update done estop = False # early stop - start_time = time.clock() + start_time = time.time() try: for eidx in xrange(max_epochs): n_samples = 0 @@ -622,7 +622,7 @@ def train_lstm( except KeyboardInterrupt: print "Training interupted" - end_time = time.clock() + end_time = time.time() if best_p is not None: zipp(best_p, tparams) else: diff --git a/code/mlp.py b/code/mlp.py index e4b95ea8..252ef868 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -336,7 +336,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, best_validation_loss = numpy.inf best_iter = 0 test_score = 0. - start_time = time.clock() + start_time = time.time() epoch = 0 done_looping = False @@ -391,7 +391,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, done_looping = True break - end_time = time.clock() + end_time = time.time() print(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i, with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) diff --git a/code/rbm.py b/code/rbm.py index 2c821fc9..bf91ced3 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -428,7 +428,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, ) plotting_time = 0. - start_time = time.clock() + start_time = time.time() # go through training epochs for epoch in xrange(training_epochs): @@ -441,7 +441,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost) # Plot filters after each training epoch - plotting_start = time.clock() + plotting_start = time.time() # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( @@ -452,10 +452,10 @@ def test_rbm(learning_rate=0.1, training_epochs=15, ) ) image.save('filters_at_epoch_%i.png' % epoch) - plotting_stop = time.clock() + plotting_stop = time.time() plotting_time += (plotting_stop - plotting_start) - end_time = time.clock() + end_time = time.time() pretraining_time = (end_time - start_time) - plotting_time From 9a0b657f7479ceb5501cfcc46734e6a4e548ecef Mon Sep 17 00:00:00 2001 From: Changxu Date: Mon, 29 Jun 2015 14:32:39 +0800 Subject: [PATCH 203/417] using timeit.default_timer would be a good solution --- code/DBN.py | 10 +++++----- code/SdA.py | 10 +++++----- code/cA.py | 6 +++--- code/convolutional_mlp.py | 6 +++--- code/dA.py | 10 +++++----- code/logistic_cg.py | 6 +++--- code/logistic_sgd.py | 6 +++--- code/mlp.py | 6 +++--- code/rbm.py | 10 +++++----- code/rnnslu.py | 6 +++--- 10 files changed, 38 insertions(+), 38 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index a18c00af..b54ac5bc 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -2,7 +2,7 @@ """ import os import sys -import time +import timeit import numpy @@ -327,7 +327,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, k=k) print '... pre-training the model' - start_time = time.time() + start_time = timeit.default_timer() ## Pre-train layer-wise for i in xrange(dbn.n_layers): # go through pretraining epochs @@ -340,7 +340,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) - end_time = time.time() + end_time = timeit.default_timer() # end-snippet-2 print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + @@ -372,7 +372,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, best_validation_loss = numpy.inf test_score = 0. - start_time = time.time() + start_time = timeit.default_timer() done_looping = False epoch = 0 @@ -424,7 +424,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, done_looping = True break - end_time = time.time() + end_time = timeit.default_timer() print( ( 'Optimization complete with best validation score of %f %%, ' diff --git a/code/SdA.py b/code/SdA.py index 95cc971a..82660e99 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -31,7 +31,7 @@ """ import os import sys -import time +import timeit import numpy @@ -379,7 +379,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, batch_size=batch_size) print '... pre-training the model' - start_time = time.time() + start_time = timeit.default_timer() ## Pre-train layer-wise corruption_levels = [.1, .2, .3] for i in xrange(sda.n_layers): @@ -394,7 +394,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), print numpy.mean(c) - end_time = time.time() + end_time = timeit.default_timer() print >> sys.stderr, ('The pretraining code for file ' + os.path.split(__file__)[1] + @@ -427,7 +427,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, best_validation_loss = numpy.inf test_score = 0. - start_time = time.time() + start_time = timeit.default_timer() done_looping = False epoch = 0 @@ -471,7 +471,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, done_looping = True break - end_time = time.time() + end_time = timeit.default_timer() print( ( 'Optimization complete with best validation score of %f %%, ' diff --git a/code/cA.py b/code/cA.py index 209cc7fa..e26a1ddf 100644 --- a/code/cA.py +++ b/code/cA.py @@ -30,7 +30,7 @@ """ import os import sys -import time +import timeit import numpy @@ -276,7 +276,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, } ) - start_time = time.time() + start_time = timeit.default_timer() ############ # TRAINING # @@ -293,7 +293,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, print 'Training epoch %d, reconstruction cost ' % epoch, numpy.mean( c_array[0]), ' jacobian norm ', numpy.mean(numpy.sqrt(c_array[1])) - end_time = time.time() + end_time = timeit.default_timer() training_time = (end_time - start_time) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index d4e996ea..b5278583 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -23,7 +23,7 @@ """ import os import sys -import time +import timeit import numpy @@ -274,7 +274,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, best_validation_loss = numpy.inf best_iter = 0 test_score = 0. - start_time = time.time() + start_time = timeit.default_timer() epoch = 0 done_looping = False @@ -326,7 +326,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, done_looping = True break - end_time = time.time() + end_time = timeit.default_timer() print('Optimization complete.') print('Best validation score of %f %% obtained at iteration %i, ' 'with test performance %f %%' % diff --git a/code/dA.py b/code/dA.py index 08ace3c3..8ea94e33 100644 --- a/code/dA.py +++ b/code/dA.py @@ -32,7 +32,7 @@ import os import sys -import time +import timeit import numpy @@ -321,7 +321,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, } ) - start_time = time.time() + start_time = timeit.default_timer() ############ # TRAINING # @@ -336,7 +336,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost ' % epoch, numpy.mean(c) - end_time = time.time() + end_time = timeit.default_timer() training_time = (end_time - start_time) @@ -379,7 +379,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, } ) - start_time = time.time() + start_time = timeit.default_timer() ############ # TRAINING # @@ -394,7 +394,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost ' % epoch, numpy.mean(c) - end_time = time.time() + end_time = timeit.default_timer() training_time = (end_time - start_time) diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 429b523a..4b4f2172 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -38,7 +38,7 @@ import os import sys -import time +import timeit import numpy @@ -275,7 +275,7 @@ def callback(theta_value): # using scipy conjugate gradient optimizer import scipy.optimize print ("Optimizing using scipy.optimize.fmin_cg...") - start_time = time.time() + start_time = timeit.default_timer() best_w_b = scipy.optimize.fmin_cg( f=train_fn, x0=numpy.zeros((n_in + 1) * n_out, dtype=x.dtype), @@ -284,7 +284,7 @@ def callback(theta_value): disp=0, maxiter=n_epochs ) - end_time = time.time() + end_time = timeit.default_timer() print( ( 'Optimization complete with best validation score of %f %%, with ' diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 1f521fe6..d197b960 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -38,7 +38,7 @@ import gzip import os import sys -import time +import timeit import numpy @@ -360,7 +360,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, best_validation_loss = numpy.inf test_score = 0. - start_time = time.time() + start_time = timeit.default_timer() done_looping = False epoch = 0 @@ -419,7 +419,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, done_looping = True break - end_time = time.time() + end_time = timeit.default_timer() print( ( 'Optimization complete with best validation score of %f %%,' diff --git a/code/mlp.py b/code/mlp.py index 252ef868..77d8002a 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -23,7 +23,7 @@ import os import sys -import time +import timeit import numpy @@ -336,7 +336,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, best_validation_loss = numpy.inf best_iter = 0 test_score = 0. - start_time = time.time() + start_time = timeit.default_timer() epoch = 0 done_looping = False @@ -391,7 +391,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, done_looping = True break - end_time = time.time() + end_time = timeit.default_timer() print(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i, with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) diff --git a/code/rbm.py b/code/rbm.py index bf91ced3..1ba4c86d 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -4,7 +4,7 @@ contain hidden variables. Restricted Boltzmann Machines further restrict BMs to those without visible-visible and hidden-hidden connections. """ -import time +import timeit try: import PIL.Image as Image @@ -428,7 +428,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, ) plotting_time = 0. - start_time = time.time() + start_time = timeit.default_timer() # go through training epochs for epoch in xrange(training_epochs): @@ -441,7 +441,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost) # Plot filters after each training epoch - plotting_start = time.time() + plotting_start = timeit.default_timer() # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( @@ -452,10 +452,10 @@ def test_rbm(learning_rate=0.1, training_epochs=15, ) ) image.save('filters_at_epoch_%i.png' % epoch) - plotting_stop = time.time() + plotting_stop = timeit.default_timer() plotting_time += (plotting_stop - plotting_start) - end_time = time.time() + end_time = timeit.default_timer() pretraining_time = (end_time - start_time) - plotting_time diff --git a/code/rnnslu.py b/code/rnnslu.py index 65363688..fad14db5 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -8,7 +8,7 @@ import stat import subprocess import sys -import time +import timeit import numpy @@ -318,13 +318,13 @@ def main(param=None): shuffle([train_lex, train_ne, train_y], param['seed']) param['ce'] = e - tic = time.time() + tic = timeit.default_timer() for i, (x, y) in enumerate(zip(train_lex, train_y)): rnn.train(x, y, param['win'], param['clr']) print '[learning] epoch %i >> %2.2f%%' % ( e, (i + 1) * 100. / nsentences), - print 'completed in %.2f (sec) <<\r' % (time.time() - tic), + print 'completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic), sys.stdout.flush() # evaluation // back into the real world : idx -> words From efa74a1a8155cbcaa87a303618dd2e73ad964be2 Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Thu, 2 Jul 2015 10:46:44 -0400 Subject: [PATCH 204/417] add input to other scripts --- code/convolutional_mlp.py | 3 +++ code/logistic_cg.py | 3 +++ code/mlp.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 0d88240d..c5092b90 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -110,6 +110,9 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): # store parameters of this layer self.params = [self.W, self.b] + # keep track of model input + self.input = input + def evaluate_lenet5(learning_rate=0.1, n_epochs=200, dataset='mnist.pkl.gz', diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 05f562a1..e2c69e87 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -97,6 +97,9 @@ def __init__(self, input, n_in, n_out): # symbolic form self.y_pred = T.argmax(self.p_y_given_x, axis=1) + # keep track of model input + self.input = input + def negative_log_likelihood(self, y): """Return the negative log-likelihood of the prediction of this model under a given target distribution. diff --git a/code/mlp.py b/code/mlp.py index e4b95ea8..17414d35 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -191,6 +191,9 @@ def __init__(self, rng, input, n_in, n_hidden, n_out): self.params = self.hiddenLayer.params + self.logRegressionLayer.params # end-snippet-3 + # keep track of model input + self.input = input + def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, dataset='mnist.pkl.gz', batch_size=20, n_hidden=500): From 24b518333b69bc57979b25598bb6a37b9d9c5875 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 2 Jul 2015 11:25:47 -0400 Subject: [PATCH 205/417] Update timing since we use MRG --- code/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 39d48f3b..cf226b42 100644 --- a/code/test.py +++ b/code/test.py @@ -86,10 +86,10 @@ def speed(): # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. - expected_times_64 = numpy.asarray([9.8, 22.5, 76.1, 73.7, 116.4, - 346.9, 381.9, 558.1, 130.4, 50.8, 113.6]) + expected_times_64 = numpy.asarray([9.8, 22.0, 76.1, 73.7, 116.4, + 346.9, 355.0, 558.1, 130.4, 50.8, 113.6]) expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, - 191.2, 226.8, 432.8, 119.5, 36.9, 78.0]) + 191.2, 199.0, 432.8, 119.5, 36.9, 78.0]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -110,7 +110,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 20.0, - 11.8, 47.9, 280.1, 132.8, 38.8, 10.5]) + 11.8, 18.2, 280.1, 132.8, 38.8, 10.5]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 07f2d5cb8e182368d7797ece9eec3e28e9eed294 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Thu, 2 Jul 2015 16:14:03 -0400 Subject: [PATCH 206/417] a link to word dictionary added --- doc/lstm.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/lstm.txt b/doc/lstm.txt index 73627c3d..34dd7d26 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -185,6 +185,8 @@ The LSTM implementation can be found in the two following files : * `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. +* `imdb.dict.pkl.gz `_ : Word-index dictionary. Use it to test a trained model on your own corpus. + After downloading both scripts and putting both in the same folder, the user can run the code by calling: From 422d801c331805c5d5593b3aeddbb4e552512515 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Thu, 2 Jul 2015 16:14:57 -0400 Subject: [PATCH 207/417] automatically download the word dictionary for lstm tutorial --- data/download.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/data/download.sh b/data/download.sh index 88e48e5a..92ef3d3c 100755 --- a/data/download.sh +++ b/data/download.sh @@ -16,6 +16,7 @@ fi $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip imdb.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.dict.pkl.gz && gunzip imdb.dict.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" $DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz From e5eb0fc28b51443ac297240728c95dcb34090f73 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Thu, 2 Jul 2015 16:28:08 -0400 Subject: [PATCH 208/417] moved the dictionary description to dataset --- doc/lstm.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 34dd7d26..a487d154 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -27,6 +27,11 @@ that has previously been preprocessed according to the needs of this LSTM implementation. Running the code provided in this tutorial will automatically download the data to the local directory. +Once the model is trained, you can test it with your own corpus using the +word-index dictionary +(`imdb.dict.pkl.gz `_) +provided as a part of this tutorial. + Model +++++ @@ -185,8 +190,6 @@ The LSTM implementation can be found in the two following files : * `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. -* `imdb.dict.pkl.gz `_ : Word-index dictionary. Use it to test a trained model on your own corpus. - After downloading both scripts and putting both in the same folder, the user can run the code by calling: From b257f1d63a3fd7b8c256dcf39418dbe483c209ff Mon Sep 17 00:00:00 2001 From: Mehdi Mirza Date: Fri, 3 Jul 2015 14:51:59 -0400 Subject: [PATCH 209/417] clarify signal --- doc/lenet.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 368b099b..3bcf1340 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -404,7 +404,7 @@ to be compatible with our previous MLP implementation. Note that the term convolution could corresponds to different mathematical operations. 1. theano.tensor.nnet.conv2d which is the most common one in almost all of the recent published convolutional models. In this op for each output feature map, all the input feature maps are summed together after being convolved with the filter. 2. Original LeNet model: In this work for each output feature map, only subset of input feature maps were selected. - 3. The convolution used in signal processing: theano.tensor.signal.conv.conv2d + 3. The convolution used in signal processing: theano.tensor.signal.conv.conv2d which works only on single channel inputs. From 76b1bb24c523e02e3aef8a2d3d5485fd6810af40 Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Mon, 6 Jul 2015 17:17:27 -0400 Subject: [PATCH 210/417] Fix formatting, extend a bit. --- doc/lenet.txt | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 3bcf1340..117dfdab 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -401,10 +401,31 @@ to be compatible with our previous MLP implementation. .. note:: - Note that the term convolution could corresponds to different mathematical operations. - 1. theano.tensor.nnet.conv2d which is the most common one in almost all of the recent published convolutional models. In this op for each output feature map, all the input feature maps are summed together after being convolved with the filter. - 2. Original LeNet model: In this work for each output feature map, only subset of input feature maps were selected. - 3. The convolution used in signal processing: theano.tensor.signal.conv.conv2d which works only on single channel inputs. + Note that the term "convolution" could corresponds to different mathematical operations: + + 1. `theano.tensor.nnet.conv2d + `_, + which is the most common one in almost all of the recent published + convolutional models. + In this operation, each output feature map is connected to each + input feature map by a different 2D filter, and its value is the sum of + the individual convolution of all inputs through the corresponding filter. + + 2. The convolution used in the original LeNet model: In this work, + each output feature map is only connected to a subset of input + feature maps. + + 3. The convolution used in signal processing: + `theano.tensor.signal.conv.conv2d + `_, + which works only on single channel inputs. + + Here, we use the first operation, so this models differ slightly + from the original LeNet paper. One reason to use 2. would be to + reduce the amount of computation needed, but modern hardware makes + it as fast to have the full connection pattern. Another reason would + be to slightly reduce the number of free parameters, but we have + other regularization techniques at our disposal. From 1c8cbc033a0a2d9f5daa9dd7344c0fcdd2c3d1ec Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Mon, 6 Jul 2015 17:17:46 -0400 Subject: [PATCH 211/417] Fix compilation warnings --- doc/logreg.txt | 2 +- doc/lstm.txt | 10 +++++----- doc/rnnslu.txt | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/logreg.txt b/doc/logreg.txt index 57065e99..c2979e63 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -266,7 +266,7 @@ instance we used a batch size of 600. Prediction Using a Trained Model -+++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++ ``sgd_optimization_mnist`` serialize and pickle the model each time new lowest validation error is reached. We can reload this model and predict diff --git a/doc/lstm.txt b/doc/lstm.txt index a487d154..71170c0b 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -211,21 +211,21 @@ If you use this tutorial, please cite the following papers. Introduction of the LSTM model: -* `[pdf] `_ Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. +* `[pdf] `__ Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. Addition of the forget gate to the LSTM model: -* `[pdf] `_ Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to forget: Continual prediction with LSTM. Neural computation, 12(10), 2451-2471. +* `[pdf] `__ Gers, F. A., Schmidhuber, J., & Cummins, F. (2000). Learning to forget: Continual prediction with LSTM. Neural computation, 12(10), 2451-2471. More recent LSTM paper: -* `[pdf] `_ Graves, Alex. Supervised sequence labelling with recurrent neural networks. Vol. 385. Springer, 2012. +* `[pdf] `__ Graves, Alex. Supervised sequence labelling with recurrent neural networks. Vol. 385. Springer, 2012. Papers related to Theano: -* `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. +* `[pdf] `__ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. -* `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. +* `[pdf] `__ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. Thank you! diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index 503d6811..cdc88723 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -27,15 +27,15 @@ Papers If you use this tutorial, cite the following papers: -* `[pdf] `_ Grégoire Mesnil, Xiaodong He, Li Deng and Yoshua Bengio. Investigation of Recurrent-Neural-Network Architectures and Learning Methods for Spoken Language Understanding. Interspeech, 2013. +* `[pdf] `__ Grégoire Mesnil, Xiaodong He, Li Deng and Yoshua Bengio. Investigation of Recurrent-Neural-Network Architectures and Learning Methods for Spoken Language Understanding. Interspeech, 2013. -* `[pdf] `_ Gokhan Tur, Dilek Hakkani-Tur and Larry Heck. What is left to be understood in ATIS? +* `[pdf] `__ Gokhan Tur, Dilek Hakkani-Tur and Larry Heck. What is left to be understood in ATIS? -* `[pdf] `_ Christian Raymond and Giuseppe Riccardi. Generative and discriminative algorithms for spoken language understanding. Interspeech, 2007. +* `[pdf] `__ Christian Raymond and Giuseppe Riccardi. Generative and discriminative algorithms for spoken language understanding. Interspeech, 2007. -* `[pdf] `_ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. +* `[pdf] `__ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. -* `[pdf] `_ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. +* `[pdf] `__ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. Thank you! From 59f919790df870c1b6dcc902d26cc411bc96e68f Mon Sep 17 00:00:00 2001 From: Pascal Lamblin Date: Mon, 6 Jul 2015 17:18:15 -0400 Subject: [PATCH 212/417] Remove stub page that has never been linked. --- doc/deep.txt | 136 --------------------------------------------------- 1 file changed, 136 deletions(-) delete mode 100644 doc/deep.txt diff --git a/doc/deep.txt b/doc/deep.txt deleted file mode 100644 index bd5e5389..00000000 --- a/doc/deep.txt +++ /dev/null @@ -1,136 +0,0 @@ -.. _deep: - -Deep Learning -============= - -The breakthrough to effective training strategies for deep architectures came in -2006 with the algorithms for training deep belief networks -(DBN) [Hinton07]_ and stacked auto-encoders [Ranzato07]_ , [Bengio07]_ . -All these methods are based on a similar approach: **greedy layer-wise unsupervised -pre-training** followed by **supervised fine-tuning**. - -The pretraining strategy consists in using unsupervised learning to guide the -training of intermediate levels of representation. Each layer is pre-trained -with an unsupervised learning algorithm, which attempts to learn a nonlinear -transformation of its input, in order to captures its main variations. Higher -levels of abstractions are created by feeding the output of one layer, to the -input of the subsequent layer. - -The resulting an architecture can then be seen in two lights: - -* the pre-trained deep network can be used to initialize the weights of all, but - the last layer of a deep neural network. The weights are then further adapted - to a supervised task (such as classification) through traditional gradient - descent (see :ref:`Multilayer perceptron `). This is referred to as the - fine-tuning step. - -* the pre-trained deep network can also serve solely as a feature extractor. The - output of the last layer is fed to a classifier, such as logistic regression, - which is trained independently. Better results can be obtained by - concatenating the output of the last layer, with the hidden representations of - all intermediate layers [Lee09]_. - -For the purposes of this tutorial, we will focus on the first interpretation, -as that is what was first proposed in [Hinton06]_. - -Deep Coding -+++++++++++ - -Since Deep Belief Networks (DBN) and Stacked Denoising-AutoEncoders (SDA) share -much of the same architecture and have very similar training algorithms (in -terms of pretraining and fine-tuning stages), it makes sense to implement them -in a similar fashion, as part of a "Deep Learning" framework. - -We thus define a generic interface, which both of these architectures will -share. - -.. code-block:: python - - class DeepLayerwiseModel(object): - - def layerwise_pretrain(self, layer_fns, pretrain_amounts): - """ - """ - - def finetune(self, datasets, lr, batch_size): - """ - - class DBN(DeepLayerwiseModel): - """ - """ - - class StackedDAA(DeepLayerwiseModel): - """ - """ - -.. code-block:: python - - def deep_main(learning_rate=0.1, - pretraining_epochs=20, - pretrain_lr=0.1, - training_epochs=1000, - batch_size=20, - mnist_file='mnist.pkl.gz'): - - n_train_examples, train_valid_test = load_mnist(mnist_file) - - # instantiate model - deep_model = ... - - #### - #### Phase 1: Pre-training - #### - - # create an array of functions, which will be used for the greedy - # layer-wise unsupervised training procedure - - pretrain_functions = deep_model.pretrain_functions( - batch_size=batch_size, - train_set_x=train_set_x, - learning_rate=pretrain_lr, - ... - ) - - # loop over all the layers in our network - for layer_idx, pretrain_fn in enumerate(pretrain_functions): - - # iterate over a certain number of epochs) - for i in xrange(pretraining_epochs * n_train_examples / batch_size): - - # follow one step in the gradient of the unsupervised cost - # function, at the given layer - layer_fn(i) - - -.. code-block:: python - - #### - #### Phase 2: Fine Tuning - #### - - # create theano functions for fine-tuning, as well as - # validation and testing our model. - - train_fn, valid_scores, test_scores =\ - deep_model.finetune_functions( - train_valid_test[0][0], # training dataset - learning_rate=finetune_lr, # the learning rate - batch_size=batch_size) # number of examples to use at once - - - # use these functions as part of the generic early-stopping procedure - for i in xrange(patience_max): - - if i >= patience: - break - - cost_i = train_fn(i) - - ... - - - - - - - From 9e278c137b677413cb716b06f62234f1d4b007b7 Mon Sep 17 00:00:00 2001 From: dallascard Date: Sun, 12 Jul 2015 17:54:53 -0400 Subject: [PATCH 213/417] fixed minor typos in gettingstarted.txt --- doc/gettingstarted.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index da82abc6..a179d291 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -104,7 +104,7 @@ MNIST Dataset Since now the data is in one variable, and a minibatch is defined as a slice of that variable, it comes more natural to define a minibatch by indicating its index and its size. In our setup the batch size stays constant - through out the execution of the code, therefore a function will actually + throughout the execution of the code, therefore a function will actually require only the index to identify on which datapoints to work. The code below shows how to store your data and how to access a minibatch: @@ -141,8 +141,8 @@ MNIST Dataset # accessing the third minibatch of the training set - data = train_set_x[2 * 500: 3 * 500] - label = train_set_y[2 * 500: 3 * 500] + data = train_set_x[2 * batch_size: 3 * batch_size] + label = train_set_y[2 * batch_size: 3 * batch_size] The data has to be stored as floats on the GPU ( the right From ca7d215c64d6412de516390f3aba69cefe219f8c Mon Sep 17 00:00:00 2001 From: dallascard Date: Sun, 12 Jul 2015 23:33:17 -0400 Subject: [PATCH 214/417] fixed a few more typos --- code/logistic_sgd.py | 6 +++--- code/mlp.py | 6 +++--- doc/rnnslu.txt | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 840616a2..75a5c4cc 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -94,11 +94,11 @@ def __init__(self, input, n_in, n_out): # symbolic expression for computing the matrix of class-membership # probabilities # Where: - # W is a matrix where column-k represent the separation hyper plain for + # W is a matrix where column-k represent the separation hyperplane for # class-k # x is a matrix where row-j represents input training sample-j - # b is a vector where element-k represent the free parameter of hyper - # plain-k + # b is a vector where element-k represent the free parameter of + # hyperplane-k self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b) # symbolic description of how to compute prediction as class whose diff --git a/code/mlp.py b/code/mlp.py index 389225b1..18f34e7c 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -296,9 +296,9 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # specify how to update the parameters of the model as a list of # (variable, update expression) pairs - # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of - # same length, zip generates a list C of same size, where each element - # is a pair formed from the two lists : + # given two lists of the same length, A = [a1, a2, a3, a4] and + # B = [b1, b2, b3, b4], zip generates a list C of same size, where each + # element is a pair formed from the two lists : # C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)] updates = [ (param, param - learning_rate * gparam) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index cdc88723..bb294c33 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -101,7 +101,7 @@ Raw input encoding A token corresponds to a word. Each token in the ATIS vocabulary is associated to an index. Each sentence is a array of indexes (``int32``). Then, each set (train, valid, test) is a list of arrays of indexes. A python -dictionnary is defined for mapping the space of indexes to the space of words. +dictionary is defined for mapping the space of indexes to the space of words. >>> sentence array([383, 189, 13, 193, 208, 307, 195, 502, 260, 539, @@ -224,7 +224,7 @@ The **parameters** of the E-RNN to be learned are: * the word embeddings (real-valued matrix) * the initial hidden state (real-value vector) * two matrices for the linear projection of the input ``t`` and the previous hidden layer state ``t-1`` -* (optionnal) bias. `Recommendation `_: don't use it. +* (optional) bias. `Recommendation `_: don't use it. * softmax classification layer on top The **hyperparameters** define the whole architecture: @@ -282,7 +282,7 @@ the true labels and compute some metrics. In this `repo `_ PERL script. It's not trivial to compute those metrics due to the `Inside Outside Beginning (IOB) `_ representation -i.e. a prediction is considered correct if the word-beginnin **and** the +i.e. a prediction is considered correct if the word-beginning **and** the word-inside **and** the word-outside predictions are **all** correct. Note that the extension is `txt` and you will have to change it to `pl`. From d323784d99b1e20c81b61df5d3f552d40778df7c Mon Sep 17 00:00:00 2001 From: Frederic Date: Mon, 13 Jul 2015 10:11:28 -0400 Subject: [PATCH 215/417] Add license file --- LICENSE.txt | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..ad9af7af --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,30 @@ +.. _license: + +LICENSE +======= + +Copyright (c) 2010--2015, Deep Learning Tutorials Development Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Theano nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 5065216475de38d13a99cd6b2df0920909738ff5 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Tue, 14 Jul 2015 11:24:15 -0400 Subject: [PATCH 216/417] preprocessing script linked --- doc/lstm.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 71170c0b..ec6ed52b 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -25,7 +25,10 @@ recurrent neural network on the Large Movie Review Dataset dataset. While the dataset is public, in this tutorial we provide a copy of the dataset that has previously been preprocessed according to the needs of this LSTM implementation. Running the code provided in this tutorial will automatically -download the data to the local directory. +download the data to the local directory. In order to use your own data, please +use a (`preprocessing script +`_) +provided as a part of this tutorial. Once the model is trained, you can test it with your own corpus using the word-index dictionary From b9a1e48aa0cb39b6619a3a6f225e2bc6b6c79682 Mon Sep 17 00:00:00 2001 From: Brandon Amos Date: Mon, 27 Jul 2015 14:47:30 -0400 Subject: [PATCH 217/417] Fix minor typo. --- code/logistic_sgd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 75a5c4cc..c944f8b3 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -81,7 +81,7 @@ def __init__(self, input, n_in, n_out): name='W', borrow=True ) - # initialize the baises b as a vector of n_out 0s + # initialize the biases b as a vector of n_out 0s self.b = theano.shared( value=numpy.zeros( (n_out,), From f899e0599ee6ac9850d5d57f1483b10b2ca2b8ca Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 28 Jul 2015 11:22:21 -0400 Subject: [PATCH 218/417] Fix typo in LSTM equations --- doc/lstm.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index ec6ed52b..51dc520d 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -124,7 +124,7 @@ output gates and, subsequently, their outputs : .. math:: :label: 5 - o_t = \sigma(W_o x_t + U_o h_{t-1} + V_o C_t + b_1) + o_t = \sigma(W_o x_t + U_o h_{t-1} + V_o C_t + b_o) .. math:: :label: 6 @@ -144,7 +144,7 @@ matrix :math:`V_o` and equation :eq:`5` is replaced by equation :eq:`5-alt` : .. math:: :label: 5-alt - o_t = \sigma(W_o x_t + U_o h_{t-1} + b_1) + o_t = \sigma(W_o x_t + U_o h_{t-1} + b_o) Our model is composed of a single LSTM layer followed by an average pooling and a logistic regression layer as illustrated in Figure 2 below. Thus, from From f97fa2399bd1eebcac26eb28a9efbef166fb0424 Mon Sep 17 00:00:00 2001 From: Pierre Luc Carrier Date: Tue, 28 Jul 2015 11:44:52 -0400 Subject: [PATCH 219/417] Add new support person for LSTM tutorial --- doc/lstm.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 51dc520d..828fd694 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -235,8 +235,9 @@ Thank you! Contact ======= -Please email `Kyunghyun Cho `_ for any -problem report or feedback. We will be glad to hear from you. +Please email `Pierre Luc Carrier `_ or +`Kyunghyun Cho `_ for any problem report or +feedback. We will be glad to hear from you. References ++++++++++ From 0c4c8c0772416fd1ff87c44044f099f75d046f0e Mon Sep 17 00:00:00 2001 From: Gustavo Serra Scalet Date: Sat, 22 Aug 2015 10:45:26 -0300 Subject: [PATCH 220/417] Fix protocol usage for cloning GitHub's repository Output on my machine: $ git clone git://github.com/lisa-lab/DeepLearningTutorials.git Cloning into 'DeepLearningTutorials'... fatal: unable to connect to github.com: github.com[0: 192.30.252.128]: errno=Connection refused However: $ git clone https://github.com/lisa-lab/DeepLearningTutorials.git Cloning into 'DeepLearningTutorials'... remote: Counting objects: 3652, done. remote: Total 3652 (delta 0), reused 0 (delta 0), pack-reused 3652 Receiving objects: 100% (3652/3652), 7.80 MiB | 1.77 MiB/s, done. Resolving deltas: 100% (2161/2161), done. Checking connectivity... done. I just used the GitHub's clone URL (default protocol is https) --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index a179d291..0a18565a 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -20,7 +20,7 @@ Download On each learning algorithm page, you will be able to download the corresponding files. If you want to download all of them at the same time, you can clone the git repository of the tutorial:: - git clone git://github.com/lisa-lab/DeepLearningTutorials.git + git clone https://github.com/lisa-lab/DeepLearningTutorials.git .. _datasets: From 0c1e636d951c832c7023ec72718f503b310252b7 Mon Sep 17 00:00:00 2001 From: Gustavo Serra Scalet Date: Sat, 22 Aug 2015 12:06:09 -0300 Subject: [PATCH 221/417] Fix typo: s/labes/label and s/recomand/recommend --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 0a18565a..5800889d 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -96,7 +96,7 @@ MNIST Dataset memory and therefore bypassing the overhead. Because the datapoints and their labels are usually of different nature (labels are usually integers while datapoints are real numbers) we - suggest to use different variables for labes and data. Also we recomand + suggest to use different variables for label and data. Also we recommend using different variables for the training set, validation set and testing set to make the code more readable (resulting in 6 different shared variables). From e9725fe7be581a7a56edcb054c53d9bcc83e1a98 Mon Sep 17 00:00:00 2001 From: Gustavo Serra Scalet Date: Sat, 22 Aug 2015 12:06:09 -0300 Subject: [PATCH 222/417] Fix typo: s/labes/label and s/recomand/recommend --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index a179d291..c0cb50bc 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -96,7 +96,7 @@ MNIST Dataset memory and therefore bypassing the overhead. Because the datapoints and their labels are usually of different nature (labels are usually integers while datapoints are real numbers) we - suggest to use different variables for labes and data. Also we recomand + suggest to use different variables for label and data. Also we recommend using different variables for the training set, validation set and testing set to make the code more readable (resulting in 6 different shared variables). From 2a6b923be3476c9eb8e2f59199bda84c77988a37 Mon Sep 17 00:00:00 2001 From: Walter Hugo Lopez Pinaya Date: Thu, 3 Sep 2015 15:05:38 -0300 Subject: [PATCH 223/417] Update hmc.txt Typo --- doc/hmc.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/hmc.txt b/doc/hmc.txt index 8348558b..c1a54cd8 100644 --- a/doc/hmc.txt +++ b/doc/hmc.txt @@ -173,7 +173,7 @@ respectively. .. literalinclude:: ../code/hmc/hmc.py :pyobject: simulate_dynamics.leapfrog -The `simulate_dynamics` function performs the full algorithm of Eqs. +The `simulate\_dynamics` function performs the full algorithm of Eqs. :eq:`leap-frog2`. We start with the initial half-step update of :math:`\phi` and full-step of :math:`s`, and then scan over the `leapfrog` method `n\_steps-1` times. @@ -232,7 +232,7 @@ defined as follows. :pyobject: kinetic_energy `hmc\_move` finally returns the tuple `(accept, final\_pos)`. `accept` is a -symbolic boolean variable indicating whether or not the new state `final_pos` +symbolic boolean variable indicating whether or not the new state `final\_pos` should be used or not. From e4cad43f96b52589e7d95a66156f698963c33e09 Mon Sep 17 00:00:00 2001 From: stray-leone Date: Sun, 13 Sep 2015 01:44:59 +0900 Subject: [PATCH 224/417] change the way of getting vocsize, nclasses. with previous way, when training data is big, need many hours. --- code/rnnslu.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 65363688..a8efda74 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -288,10 +288,8 @@ def main(param=None): valid_lex, valid_ne, valid_y = valid_set test_lex, test_ne, test_y = test_set - vocsize = len(set(reduce(lambda x, y: list(x) + list(y), - train_lex + valid_lex + test_lex))) - nclasses = len(set(reduce(lambda x, y: list(x)+list(y), - train_y + test_y + valid_y))) + vocsize = len(dic['words2idx']) + nclasses = len(dic['labels2idx']) nsentences = len(train_lex) groundtruth_valid = [map(lambda x: idx2label[x], y) for y in valid_y] From 7c1219dabbb24ea62d74ae90b2a39e9ad0c9a090 Mon Sep 17 00:00:00 2001 From: Joakim Skarding Date: Sat, 24 Oct 2015 20:50:42 +0200 Subject: [PATCH 225/417] Added MSGD abbreviation introduction --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 5800889d..63f93597 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -389,7 +389,7 @@ form, we estimate the gradient from just a single example at a time. The variant that we recommend for deep learning is a further twist on stochastic gradient descent using so-called "minibatches". -Minibatch SGD works identically to SGD, except that we use more than +Minibatch SGD (MSGD) works identically to SGD, except that we use more than one training example to make each estimate of the gradient. This technique reduces variance in the estimate of the gradient, and often makes better use of the hierarchical memory organization in modern computers. From 321a6e1adf3d650e8393f98d65f2faa67ea27f45 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 28 Oct 2015 10:37:21 -0400 Subject: [PATCH 226/417] Update atis URL --- data/download.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data/download.sh b/data/download.sh index 92ef3d3c..ed273bbb 100755 --- a/data/download.sh +++ b/data/download.sh @@ -19,8 +19,8 @@ $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip imdb.p $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.dict.pkl.gz && gunzip imdb.dict.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" -$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold0.pkl.gz -$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold1.pkl.gz -$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold2.pkl.gz -$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold3.pkl.gz -$DL_CMD http://www-etud.iro.umontreal.ca/~mesnilgr/atis/atis.fold4.pkl.gz +$DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold0.pkl.gz +$DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold1.pkl.gz +$DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold2.pkl.gz +$DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold3.pkl.gz +$DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold4.pkl.gz From 95cfe74621f3306cc7f6c0d610411054b98e91e6 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 28 Oct 2015 11:00:21 -0400 Subject: [PATCH 227/417] Fix travis, newer scipy version in miniconda have problems with gfortran --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7873dedf..4344a63d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - conda update --yes conda install: - - conda create --yes -q -n pyenv mkl python=2.7 numpy scipy pip nose yaml pyflakes pillow pyparsing=1.5 + - conda create --yes -q -n pyenv mkl python=2.7 numpy=1.10 scipy=0.16.0 pip nose yaml pyflakes pillow pyparsing=1.5 - source activate pyenv - pip install git+git://github.com/Theano/Theano.git From b3c4a9e2f7630332f802e9acc2fe1f9452920e49 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 28 Oct 2015 11:41:19 -0400 Subject: [PATCH 228/417] Use the new travis infrastucture --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4344a63d..17e75146 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ # After changing this file, check it on: # http://lint.travis-ci.org/ +sudo: false language: python #python: From 1f628ff222d7f864559bcb62a892cb0dfcb5cb65 Mon Sep 17 00:00:00 2001 From: Frederic Date: Wed, 28 Oct 2015 13:31:23 -0400 Subject: [PATCH 229/417] Update Gregoire email --- doc/rnnslu.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/rnnslu.txt b/doc/rnnslu.txt index bb294c33..7fef1683 100644 --- a/doc/rnnslu.txt +++ b/doc/rnnslu.txt @@ -42,8 +42,9 @@ Thank you! Contact ======= -Please email to `Grégoire Mesnil `_ for any -problem report or feedback. We will be glad to hear from you. +Please email to +``Grégoire Mesnil (first-add-a-dot-last-add-at-gmail-add-a-dot-com)`` +for any problem report or feedback. We will be glad to hear from you. Task ++++ From 5b62a38639200960ba58ad39bcb1dd60370b68b2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 29 Oct 2015 16:28:23 -0400 Subject: [PATCH 230/417] Small update to lstm code. uidx have already been incremented fix gh-122 --- code/lstm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index b64970fb..1d87cfb3 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -569,7 +569,7 @@ def train_lstm( f_update(lrate) if numpy.isnan(cost) or numpy.isinf(cost): - print 'NaN detected' + print 'bad cost detected: ', cost return 1., 1., 1. if numpy.mod(uidx, dispFreq) == 0: @@ -595,7 +595,7 @@ def train_lstm( history_errs.append([valid_err, test_err]) - if (uidx == 0 or + if (best_p is None or valid_err <= numpy.array(history_errs)[:, 0].min()): From 8b1e2b35f6f67017d76a4c0b7600132cf07ce5f8 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 29 Oct 2015 16:29:46 -0400 Subject: [PATCH 231/417] Update timming with speed up --- code/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/test.py b/code/test.py index cf226b42..94e03b8d 100644 --- a/code/test.py +++ b/code/test.py @@ -87,7 +87,7 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.8, 22.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 558.1, 130.4, 50.8, 113.6]) + 346.9, 355.0, 558.1, 130.4, 23.2, 106]) expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, 191.2, 199.0, 432.8, 119.5, 36.9, 78.0]) @@ -110,7 +110,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 20.0, - 11.8, 18.2, 280.1, 132.8, 38.8, 10.5]) + 11.2, 17.2, 257.7, 118.8, 34.2, 8.7]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 564af7e8381843b368b3fde86ef2d8ce13152cb2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Thu, 5 Nov 2015 07:54:38 -0500 Subject: [PATCH 232/417] Fix miniconda path due to having changed --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 17e75146..8a84a9ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b - - export PATH=/home/travis/miniconda/bin:$PATH + - export PATH=/home/travis/miniconda/bin:/home/travis/miniconda2/bin:$PATH - conda update --yes conda install: From 74ab26817a1b0a04371a25a7109d7779f7f21b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20=C4=B0skender=20Turan?= Date: Tue, 15 Dec 2015 16:56:10 +0430 Subject: [PATCH 233/417] Update gettingstarted.txt L2_sqr variable must be L2 for the loss function or loss function L2 variable must be L2_sqr --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 63f93597..e838d706 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -525,7 +525,7 @@ L2 regularization term weighted by :math:`\lambda_2` L1 = T.sum(abs(param)) # symbolic Theano variable that represents the squared L2 term - L2_sqr = T.sum(param ** 2) + L2 = T.sum(param ** 2) # the loss loss = NLL + lambda_1 * L1 + lambda_2 * L2 From fb5e394b25d4ce0d7a01a776045dc15b93697bf2 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Dec 2015 11:31:10 -0500 Subject: [PATCH 234/417] Update timing that is now faster --- code/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 94e03b8d..41749231 100644 --- a/code/test.py +++ b/code/test.py @@ -87,9 +87,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.8, 22.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 558.1, 130.4, 23.2, 106]) - expected_times_32 = numpy.asarray([8.1, 17.9, 42.5, 66.5, 71, - 191.2, 199.0, 432.8, 119.5, 36.9, 78.0]) + 346.9, 355.0, 510.9, 130.4, 23.2, 106]) + expected_times_32 = numpy.asarray([6.4, 17.9, 42.5, 66.5, 71, + 191.2, 199.0, 400.4, 119.5, 36.9, 67.2]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -110,7 +110,7 @@ def speed(): expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, 5.8, 20.0, - 11.2, 17.2, 257.7, 118.8, 34.2, 8.7]) + 11.2, 17.2, 244.3, 118.8, 34.2, 8.7]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From e9711aaf2e059ad7beceb2c762b5729ec0de7f68 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Dec 2015 11:46:34 -0500 Subject: [PATCH 235/417] Try to fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a84a9ea..4528a5fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: - conda update --yes conda install: - - conda create --yes -q -n pyenv mkl python=2.7 numpy=1.10 scipy=0.16.0 pip nose yaml pyflakes pillow pyparsing=1.5 + - conda create --yes -q -n pyenv mkl python=2.7 numpy=1.10 scipy=0.16.1 pip nose yaml pyflakes pillow pyparsing=1.5 - source activate pyenv - pip install git+git://github.com/Theano/Theano.git From e9fc9c395d7fd3b8dafafb5229e4660154727128 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Dec 2015 12:10:41 -0500 Subject: [PATCH 236/417] Add test_lstm in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4528a5fe..ae3801c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" - - PART="test.py:test_SdA" + - PART="test.py:test_SdA test.py:test_lstm" - PART="test.py:test_dbn" - PART="test.py:test_rbm test.py:test_rnnrbm" - PART="-e test.py" From 407fd81b1a5bf8630187eddabe3ff1e22afa6ca7 Mon Sep 17 00:00:00 2001 From: Frederic Date: Tue, 15 Dec 2015 12:10:56 -0500 Subject: [PATCH 237/417] Get rid of scipy dependency by using numpy. --- code/hmc/test_hmc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/code/hmc/test_hmc.py b/code/hmc/test_hmc.py index 0a70190a..f6c3b522 100644 --- a/code/hmc/test_hmc.py +++ b/code/hmc/test_hmc.py @@ -1,5 +1,4 @@ import numpy -from scipy import linalg import theano from hmc import HMC_sampler @@ -15,7 +14,7 @@ def sampler_on_nd_gaussian(sampler_cls, burnin, n_samples, dim=10): cov = numpy.array(rng.rand(dim, dim), dtype=theano.config.floatX) cov = (cov + cov.T) / 2. cov[numpy.arange(dim), numpy.arange(dim)] = 1.0 - cov_inv = linalg.inv(cov) + cov_inv = numpy.linalg.inv(cov) # Define energy function for a multi-variate Gaussian def gaussian_energy(x): From e1d1e0c826d7d05a84302222d55b45cf3af2a6fa Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 15 Jan 2016 16:23:24 -0500 Subject: [PATCH 238/417] Update timing for case that is now faster --- code/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/test.py b/code/test.py index 41749231..76c95b38 100644 --- a/code/test.py +++ b/code/test.py @@ -86,9 +86,9 @@ def speed(): # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. - expected_times_64 = numpy.asarray([9.8, 22.0, 76.1, 73.7, 116.4, + expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, 346.9, 355.0, 510.9, 130.4, 23.2, 106]) - expected_times_32 = numpy.asarray([6.4, 17.9, 42.5, 66.5, 71, + expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 66.5, 71, 191.2, 199.0, 400.4, 119.5, 36.9, 67.2]) # Number with just 1 decimal are new value that are faster with From b3b1783b3fb1ac92f28dbc3a4e64c5ea7cf85731 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 15 Jan 2016 16:39:06 -0500 Subject: [PATCH 239/417] Add test_rnnslu to travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ae3801c2..258963ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ env: - PART="test.py:test_logistic_sgd test.py:test_logistic_cg test.py:test_mlp test.py:test_convolutional_mlp test.py:test_dA" - PART="test.py:test_SdA test.py:test_lstm" - PART="test.py:test_dbn" - - PART="test.py:test_rbm test.py:test_rnnrbm" + - PART="test.py:test_rbm test.py:test_rnnrbm test.py:test_rnnslu" - PART="-e test.py" #i7-2600K CPU @ 3.40GHz From 1a1529261e05fb5d27be973439c5cb4f2ce49d94 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 15 Jan 2016 16:39:35 -0500 Subject: [PATCH 240/417] Make test_rnnslu faster --- code/test.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 76c95b38..ff2ff359 100644 --- a/code/test.py +++ b/code/test.py @@ -15,10 +15,6 @@ import lstm -def test_rnnslu(): - rnnslu.main() - - def test_logistic_sgd(): logistic_sgd.sgd_optimization_mnist(n_epochs=10) @@ -62,6 +58,26 @@ def test_rnnrbm(): rnnrbm.test_rnnrbm(num_epochs=1) +def test_rnnslu(): + s = {'fold': 3, + # 5 folds 0,1,2,3,4 + 'data': 'atis', + 'lr': 0.0970806646812754, + 'verbose': 1, + 'decay': True, + # decay on the learning rate if improvement stops + 'win': 7, + # number of words in the context window + 'nhidden': 200, + # number of hidden units + 'seed': 345, + 'emb_dimension': 50, + # dimension of word embedding + 'nepochs': 1, # CHANGED + 'savemodel': False} + rnnslu.main(s) + + def test_lstm(): lstm.train_lstm(max_epochs=1, test_size=1000, saveto='') From 87b2f9a1d22757681c6c4636bd8d9219ba91cd7d Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 15 Jan 2016 16:41:35 -0500 Subject: [PATCH 241/417] Small diff to help debug by having better error message --- code/rnnslu.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 2ea55978..2251e465 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -126,11 +126,14 @@ def get_perf(filename, folder): stdout=subprocess.PIPE) stdout, _ = proc.communicate(''.join(open(filename).readlines())) + out = None for line in stdout.split('\n'): if 'accuracy' in line: out = line.split() break - + # To help debug + if out is None: + print stdout.split('\n') precision = float(out[6][:-2]) recall = float(out[8][:-2]) f1score = float(out[10]) From b701733044d73681baa8346973229ed8d0537395 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 27 Jan 2016 21:40:33 -0800 Subject: [PATCH 242/417] Use the In object as Param is deprecated --- code/DBN.py | 2 +- code/SdA.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index b54ac5bc..ecd563e7 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -174,7 +174,7 @@ def pretraining_functions(self, train_set_x, batch_size, k): # compile the theano function fn = theano.function( - inputs=[index, theano.Param(learning_rate, default=0.1)], + inputs=[index, theano.In(learning_rate, value=0.1)], outputs=cost, updates=updates, givens={ diff --git a/code/SdA.py b/code/SdA.py index 82660e99..c74c2986 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -214,8 +214,8 @@ def pretraining_functions(self, train_set_x, batch_size): fn = theano.function( inputs=[ index, - theano.Param(corruption_level, default=0.2), - theano.Param(learning_rate, default=0.1) + theano.In(corruption_level, value=0.2), + theano.In(learning_rate, value=0.1) ], outputs=cost, updates=updates, From 6e3d61544f2786f7400a98151db99c5409c8bb4e Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Tue, 2 Feb 2016 16:41:10 -0500 Subject: [PATCH 243/417] Update convolution to use the updated interface. --- code/convolutional_mlp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 64bf5e69..bb6aeaf4 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -30,7 +30,7 @@ import theano import theano.tensor as T from theano.tensor.signal import downsample -from theano.tensor.nnet import conv +from theano.tensor.nnet import conv2d from logistic_sgd import LogisticRegression, load_data from mlp import HiddenLayer @@ -87,7 +87,7 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): self.b = theano.shared(value=b_values, borrow=True) # convolve input feature maps with filters - conv_out = conv.conv2d( + conv_out = conv2d( input=input, filters=self.W, filter_shape=filter_shape, From 6c5f07bbdbfcc9ed8fb6c8ae05b288a5ce696a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Bastien?= Date: Wed, 3 Feb 2016 12:01:42 -0500 Subject: [PATCH 244/417] Update timing that got speed up. --- code/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/test.py b/code/test.py index 76c95b38..8b6a515e 100644 --- a/code/test.py +++ b/code/test.py @@ -87,7 +87,7 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 510.9, 130.4, 23.2, 106]) + 346.9, 355.0, 510.9, 130.4, 23.2, 98.8]) expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 66.5, 71, 191.2, 199.0, 400.4, 119.5, 36.9, 67.2]) @@ -108,7 +108,7 @@ def speed(): #expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138 # 1.35324519 1.7356905 1.12937868] - expected_times_gpu = numpy.asarray([3.0, 7.55523491, 18.99226785, + expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, 5.8, 20.0, 11.2, 17.2, 244.3, 118.8, 34.2, 8.7]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) From c26252342c4d1ef1fea9131c7605d4190e52b2c2 Mon Sep 17 00:00:00 2001 From: Benjamin Irving Date: Wed, 3 Feb 2016 17:56:49 +0000 Subject: [PATCH 245/417] fix minor typos and formatting --- code/logistic_sgd.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index c944f8b3..68f26911 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -207,12 +207,12 @@ def load_data(dataset): f = gzip.open(dataset, 'rb') train_set, valid_set, test_set = cPickle.load(f) f.close() - #train_set, valid_set, test_set format: tuple(input, target) - #input is an numpy.ndarray of 2 dimensions (a matrix) - #witch row's correspond to an example. target is a - #numpy.ndarray of 1 dimensions (vector)) that have the same length as - #the number of rows in the input. It should give the target - #target to the example with the same index in the input. + # train_set, valid_set, test_set format: tuple(input, target) + # input is a numpy.ndarray of 2 dimensions (a matrix) + # where each row corresponds to an example. target is a + # numpy.ndarray of 1 dimension (vector) that has the same length as + # the number of rows in the input. It should give the target + # to the example with the same index in the input. def shared_dataset(data_xy, borrow=True): """ Function that loads the dataset into shared variables From d2764f288b4e58e12bd492953d1c1a0b43d92e21 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Thu, 21 Jan 2016 11:06:00 -0500 Subject: [PATCH 246/417] successfully ported logistic_sgd.py --- code/logistic_sgd.py | 59 ++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/code/logistic_sgd.py b/code/logistic_sgd.py index 68f26911..9f4427e7 100644 --- a/code/logistic_sgd.py +++ b/code/logistic_sgd.py @@ -32,9 +32,12 @@ Christopher M. Bishop, section 4.3.2 """ + +from __future__ import print_function + __docformat__ = 'restructedtext en' -import cPickle +import six.moves.cPickle as pickle import gzip import os import sys @@ -194,19 +197,21 @@ def load_data(dataset): dataset = new_path if (not os.path.isfile(dataset)) and data_file == 'mnist.pkl.gz': - import urllib + from six.moves import urllib origin = ( 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz' ) - print 'Downloading data from %s' % origin - urllib.urlretrieve(origin, dataset) + print('Downloading data from %s' % origin) + urllib.request.urlretrieve(origin, dataset) - print '... loading data' + print('... loading data') # Load the dataset - f = gzip.open(dataset, 'rb') - train_set, valid_set, test_set = cPickle.load(f) - f.close() + with gzip.open(dataset, 'rb') as f: + try: + train_set, valid_set, test_set = pickle.load(f, encoding='latin1') + except: + train_set, valid_set, test_set = pickle.load(f) # train_set, valid_set, test_set format: tuple(input, target) # input is a numpy.ndarray of 2 dimensions (a matrix) # where each row corresponds to an example. target is a @@ -276,14 +281,14 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size + n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size + n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size ###################### # BUILD ACTUAL MODEL # ###################### - print '... building the model' + print('... building the model') # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch @@ -348,14 +353,14 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, ############### # TRAIN MODEL # ############### - print '... training the model' + print('... training the model') # early-stopping parameters patience = 5000 # look as this many examples regardless patience_increase = 2 # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant - validation_frequency = min(n_train_batches, patience / 2) + validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we @@ -369,7 +374,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, epoch = 0 while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_model(minibatch_index) # iteration number @@ -378,7 +383,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, if (iter + 1) % validation_frequency == 0: # compute zero-one loss on validation set validation_losses = [validate_model(i) - for i in xrange(n_valid_batches)] + for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print( @@ -402,7 +407,7 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, # test it on the test set test_losses = [test_model(i) - for i in xrange(n_test_batches)] + for i in range(n_test_batches)] test_score = numpy.mean(test_losses) print( @@ -419,8 +424,8 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, ) # save the best model - with open('best_model.pkl', 'w') as f: - cPickle.dump(classifier, f) + with open('best_model.pkl', 'wb') as f: + pickle.dump(classifier, f) if patience <= iter: done_looping = True @@ -434,11 +439,11 @@ def sgd_optimization_mnist(learning_rate=0.13, n_epochs=1000, ) % (best_validation_loss * 100., test_score * 100.) ) - print 'The code run for %d epochs, with %f epochs/sec' % ( - epoch, 1. * epoch / (end_time - start_time)) - print >> sys.stderr, ('The code for file ' + - os.path.split(__file__)[1] + - ' ran for %.1fs' % ((end_time - start_time))) + print('The code run for %d epochs, with %f epochs/sec' % ( + epoch, 1. * epoch / (end_time - start_time))) + print(('The code for file ' + + os.path.split(__file__)[1] + + ' ran for %.1fs' % ((end_time - start_time))), file=sys.stderr) def predict(): @@ -448,7 +453,7 @@ def predict(): """ # load the saved model - classifier = cPickle.load(open('best_model.pkl')) + classifier = pickle.load(open('best_model.pkl')) # compile a predictor function predict_model = theano.function( @@ -462,8 +467,8 @@ def predict(): test_set_x = test_set_x.get_value() predicted_values = predict_model(test_set_x[:10]) - print ("Predicted values for the first 10 examples in test set:") - print predicted_values + print("Predicted values for the first 10 examples in test set:") + print(predicted_values) if __name__ == '__main__': From 2c022d15401c67538fabeb1b5ae2a7470d5fb2f2 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Thu, 21 Jan 2016 15:28:29 -0500 Subject: [PATCH 247/417] fixed everything except rnnrbm and rnnslu, partial tests run but not to completion --- code/SdA.py | 51 +++++++++++++----------- code/cA.py | 16 +++++--- code/convolutional_mlp.py | 31 ++++++++------- code/dA.py | 28 +++++++------ code/hmc/hmc.py | 6 +-- code/hmc/test_hmc.py | 29 +++++++++----- code/imdb.py | 17 +++++--- code/lstm.py | 83 +++++++++++++++++++++------------------ code/mlp.py | 27 +++++++------ code/rbm.py | 10 +++-- code/utils.py | 1 + 11 files changed, 169 insertions(+), 130 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index c74c2986..d639cb54 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -29,6 +29,9 @@ Systems 19, 2007 """ + +from __future__ import print_function + import os import sys import timeit @@ -116,7 +119,7 @@ def __init__( # stochastich gradient descent on the MLP # start-snippet-2 - for i in xrange(self.n_layers): + for i in range(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden units of @@ -254,9 +257,9 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): # compute number of minibatches for training, validation and testing n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] - n_valid_batches /= batch_size + n_valid_batches //= batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] - n_test_batches /= batch_size + n_test_batches //= batch_size index = T.lscalar('index') # index to a [mini]batch @@ -314,11 +317,11 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): # Create a function that scans the entire validation set def valid_score(): - return [valid_score_i(i) for i in xrange(n_valid_batches)] + return [valid_score_i(i) for i in range(n_valid_batches)] # Create a function that scans the entire test set def test_score(): - return [test_score_i(i) for i in xrange(n_test_batches)] + return [test_score_i(i) for i in range(n_test_batches)] return train_fn, valid_score, test_score @@ -357,12 +360,12 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] - n_train_batches /= batch_size + n_train_batches //= batch_size # numpy random generator # start-snippet-3 numpy_rng = numpy.random.RandomState(89677) - print '... building the model' + print('... building the model') # construct the stacked denoising autoencoder class sda = SdA( numpy_rng=numpy_rng, @@ -374,52 +377,52 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, ######################### # PRETRAINING THE MODEL # ######################### - print '... getting the pretraining functions' + print('... getting the pretraining functions') pretraining_fns = sda.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size) - print '... pre-training the model' + print('... pre-training the model') start_time = timeit.default_timer() ## Pre-train layer-wise corruption_levels = [.1, .2, .3] - for i in xrange(sda.n_layers): + for i in range(sda.n_layers): # go through pretraining epochs - for epoch in xrange(pretraining_epochs): + for epoch in range(pretraining_epochs): # go through the training set c = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) - print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), - print numpy.mean(c) + print('Pre-training layer %i, epoch %d, cost ' % (i, epoch)) + print(numpy.mean(c)) end_time = timeit.default_timer() - print >> sys.stderr, ('The pretraining code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) / 60.)) + print(('The pretraining code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr) # end-snippet-4 ######################## # FINETUNING THE MODEL # ######################## # get the training, validation and testing function for the model - print '... getting the finetuning functions' + print('... getting the finetuning functions') train_fn, validate_model, test_model = sda.build_finetune_functions( datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr ) - print '... finetunning the model' + print('... finetunning the model') # early-stopping parameters patience = 10 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant - validation_frequency = min(n_train_batches, patience / 2) + validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we @@ -434,7 +437,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index @@ -480,9 +483,9 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.) ) - print >> sys.stderr, ('The training code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) / 60.)) + print(('The training code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr) if __name__ == '__main__': diff --git a/code/cA.py b/code/cA.py index e26a1ddf..0d563ef2 100644 --- a/code/cA.py +++ b/code/cA.py @@ -28,6 +28,10 @@ Systems 19, 2007 """ + +from __future__ import print_function +from six.moves import xrange + import os import sys import timeit @@ -205,7 +209,7 @@ def get_cost_updates(self, contraction_level, learning_rate): axis=1) # Compute the jacobian and average over the number of samples/minibatch - self.L_jacob = T.sum(J ** 2) / self.n_batchsize + self.L_jacob = T.sum(J ** 2) // self.n_batchsize # note : L is now a vector, where each element is the # cross-entropy cost of the reconstruction of the @@ -246,7 +250,7 @@ def test_cA(learning_rate=0.01, training_epochs=20, train_set_x, train_set_y = datasets[0] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch @@ -290,15 +294,15 @@ def test_cA(learning_rate=0.01, training_epochs=20, c.append(train_ca(batch_index)) c_array = numpy.vstack(c) - print 'Training epoch %d, reconstruction cost ' % epoch, numpy.mean( - c_array[0]), ' jacobian norm ', numpy.mean(numpy.sqrt(c_array[1])) + print('Training epoch %d, reconstruction cost ' % epoch, numpy.mean( + c_array[0]), ' jacobian norm ', numpy.mean(numpy.sqrt(c_array[1]))) end_time = timeit.default_timer() training_time = (end_time - start_time) - print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] + - ' ran for %.2fm' % ((training_time) / 60.)) + print(('The code for file ' + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((training_time) / 60.)), file=sys.stderr) image = Image.fromarray(tile_raster_images( X=ca.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index bb6aeaf4..a8811bc1 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -21,6 +21,9 @@ http://yann.lecun.com/exdb/publis/pdf/lecun-98.pdf """ + +from __future__ import print_function + import os import sys import timeit @@ -70,7 +73,7 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): # each unit in the lower layer receives a gradient from: # "num output feature maps * filter height * filter width" / # pooling size - fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / + fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) // numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) @@ -145,9 +148,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, n_train_batches = train_set_x.get_value(borrow=True).shape[0] n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] n_test_batches = test_set_x.get_value(borrow=True).shape[0] - n_train_batches /= batch_size - n_valid_batches /= batch_size - n_test_batches /= batch_size + n_train_batches //= batch_size + n_valid_batches //= batch_size + n_test_batches //= batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch @@ -160,7 +163,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, ###################### # BUILD ACTUAL MODEL # ###################### - print '... building the model' + print('... building the model') # Reshape matrix of rasterized images of shape (batch_size, 28 * 28) # to a 4D tensor, compatible with our LeNetConvPoolLayer @@ -261,14 +264,14 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, ############### # TRAIN MODEL # ############### - print '... training' + print('... training') # early-stopping parameters patience = 10000 # look as this many examples regardless patience_increase = 2 # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant - validation_frequency = min(n_train_batches, patience / 2) + validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we @@ -284,19 +287,19 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): iter = (epoch - 1) * n_train_batches + minibatch_index if iter % 100 == 0: - print 'training @ iter = ', iter + print('training @ iter = ', iter) cost_ij = train_model(minibatch_index) if (iter + 1) % validation_frequency == 0: # compute zero-one loss on validation set validation_losses = [validate_model(i) for i - in xrange(n_valid_batches)] + in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, @@ -317,7 +320,7 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, # test it on the test set test_losses = [ test_model(i) - for i in xrange(n_test_batches) + for i in range(n_test_batches) ] test_score = numpy.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' @@ -334,9 +337,9 @@ def evaluate_lenet5(learning_rate=0.1, n_epochs=200, print('Best validation score of %f %% obtained at iteration %i, ' 'with test performance %f %%' % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) - print >> sys.stderr, ('The code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) / 60.)) + print(('The code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr) if __name__ == '__main__': evaluate_lenet5() diff --git a/code/dA.py b/code/dA.py index 8ea94e33..0d9efa54 100644 --- a/code/dA.py +++ b/code/dA.py @@ -30,6 +30,8 @@ """ +from __future__ import print_function + import os import sys import timeit @@ -280,7 +282,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, train_set_x, train_set_y = datasets[0] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size # start-snippet-2 # allocate symbolic variables for the data @@ -328,21 +330,21 @@ def test_dA(learning_rate=0.1, training_epochs=15, ############ # go through training epochs - for epoch in xrange(training_epochs): + for epoch in range(training_epochs): # go through trainng set c = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): c.append(train_da(batch_index)) - print 'Training epoch %d, cost ' % epoch, numpy.mean(c) + print('Training epoch %d, cost ' % epoch, numpy.mean(c)) end_time = timeit.default_timer() training_time = (end_time - start_time) - print >> sys.stderr, ('The no corruption code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((training_time) / 60.)) + print(('The no corruption code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((training_time) / 60.)), file=sys.stderr) image = Image.fromarray( tile_raster_images(X=da.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(10, 10), @@ -386,21 +388,21 @@ def test_dA(learning_rate=0.1, training_epochs=15, ############ # go through training epochs - for epoch in xrange(training_epochs): + for epoch in range(training_epochs): # go through trainng set c = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): c.append(train_da(batch_index)) - print 'Training epoch %d, cost ' % epoch, numpy.mean(c) + print('Training epoch %d, cost ' % epoch, numpy.mean(c)) end_time = timeit.default_timer() training_time = (end_time - start_time) - print >> sys.stderr, ('The 30% corruption code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % (training_time / 60.)) + print(('The 30% corruption code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % (training_time / 60.)), file=sys.stderr) # end-snippet-3 # start-snippet-4 diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index b9c872f0..aeb49937 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -128,14 +128,14 @@ def leapfrog(pos, vel, step): rval2: dictionary Dictionary of updates for the Scan Op """ - # from pos(t) and vel(t-stepsize/2), compute vel(t+stepsize/2) + # from pos(t) and vel(t-stepsize//2), compute vel(t+stepsize//2) dE_dpos = TT.grad(energy_fn(pos).sum(), pos) new_vel = vel - step * dE_dpos - # from vel(t+stepsize/2) compute pos(t+stepsize) + # from vel(t+stepsize//2) compute pos(t+stepsize) new_pos = pos + step * new_vel return [new_pos, new_vel], {} - # compute velocity at time-step: t + stepsize/2 + # compute velocity at time-step: t + stepsize//2 initial_energy = energy_fn(initial_pos) dE_dpos = TT.grad(initial_energy.sum(), initial_pos) vel_half_step = initial_vel - 0.5 * stepsize * dE_dpos diff --git a/code/hmc/test_hmc.py b/code/hmc/test_hmc.py index f6c3b522..c3c425e6 100644 --- a/code/hmc/test_hmc.py +++ b/code/hmc/test_hmc.py @@ -1,7 +1,16 @@ + +from __future__ import print_function +from six.moves import xrange + import numpy import theano -from hmc import HMC_sampler +try: + from hmc import HMC_sampler +except: + # python 3 compatibility + # http://stackoverflow.com/questions/3073259/python-nose-import-error + from hmc.hmc import HMC_sampler def sampler_on_nd_gaussian(sampler_cls, burnin, n_samples, dim=10): @@ -37,17 +46,17 @@ def gaussian_energy(x): # Flatten to [n_samples * batchsize, dim] samples = _samples.T.reshape(dim, -1).T - print '****** TARGET VALUES ******' - print 'target mean:', mu - print 'target cov:\n', cov + print('****** TARGET VALUES ******') + print('target mean:', mu) + print('target cov:\n', cov) - print '****** EMPIRICAL MEAN/COV USING HMC ******' - print 'empirical mean: ', samples.mean(axis=0) - print 'empirical_cov:\n', numpy.cov(samples.T) + print('****** EMPIRICAL MEAN/COV USING HMC ******') + print('empirical mean: ', samples.mean(axis=0)) + print('empirical_cov:\n', numpy.cov(samples.T)) - print '****** HMC INTERNALS ******' - print 'final stepsize', sampler.stepsize.get_value() - print 'final acceptance_rate', sampler.avg_acceptance_rate.get_value() + print('****** HMC INTERNALS ******') + print('final stepsize', sampler.stepsize.get_value()) + print('final acceptance_rate', sampler.avg_acceptance_rate.get_value()) return sampler diff --git a/code/imdb.py b/code/imdb.py index 21e0e376..341be231 100644 --- a/code/imdb.py +++ b/code/imdb.py @@ -1,4 +1,7 @@ -import cPickle +from __future__ import print_function +from six.moves import xrange +import six.moves.cPickle as pickle + import gzip import os @@ -68,9 +71,11 @@ def get_dataset_file(dataset, default_dataset, origin): dataset = new_path if (not os.path.isfile(dataset)) and data_file == default_dataset: - import urllib - print 'Downloading data from %s' % origin - urllib.urlretrieve(origin, dataset) + from six.moves import urllib + print('Downloading data from %s' % origin) + urllib.request.urlretrieve(origin, dataset) + + return dataset @@ -110,8 +115,8 @@ def load_data(path="imdb.pkl", n_words=100000, valid_portion=0.1, maxlen=None, else: f = open(path, 'rb') - train_set = cPickle.load(f) - test_set = cPickle.load(f) + train_set = pickle.load(f) + test_set = pickle.load(f) f.close() if maxlen: new_train_set_x = [] diff --git a/code/lstm.py b/code/lstm.py index 1d87cfb3..b3b89f3e 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -1,8 +1,13 @@ ''' Build a tweet sentiment analyzer ''' + +from __future__ import print_function +from six.moves import xrange +import six.moves.cPickle as pickle + +#from six.moves.collections import OrderedDict from collections import OrderedDict -import cPickle as pkl import sys import time @@ -56,7 +61,7 @@ def zipp(params, tparams): """ When we reload the model. Needed for the GPU stuff. """ - for kk, vv in params.iteritems(): + for kk, vv in params.items(): tparams[kk].set_value(vv) @@ -65,7 +70,7 @@ def unzip(zipped): When we pickle the model. Needed for the GPU stuff. """ new_params = OrderedDict() - for kk, vv in zipped.iteritems(): + for kk, vv in zipped.items(): new_params[kk] = vv.get_value() return new_params @@ -106,7 +111,7 @@ def init_params(options): def load_params(path, params): pp = numpy.load(path) - for kk, vv in params.iteritems(): + for kk, vv in params.items(): if kk not in pp: raise Warning('%s is not in the archive' % kk) params[kk] = pp[kk] @@ -116,7 +121,7 @@ def load_params(path, params): def init_tparams(params): tparams = OrderedDict() - for kk, pp in params.iteritems(): + for kk, pp in params.items(): tparams[kk] = theano.shared(params[kk], name=kk) return tparams @@ -217,7 +222,7 @@ def sgd(lr, tparams, grads, x, mask, y, cost): # New set of shared variable that will contain the gradient # for a mini-batch. gshared = [theano.shared(p.get_value() * 0., name='%s_grad' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] gsup = [(gs, g) for gs, g in zip(gshared, grads)] # Function that computes gradients for a mini-batch, but do not @@ -266,13 +271,13 @@ def adadelta(lr, tparams, grads, x, mask, y, cost): zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] running_up2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rup2' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rg2up = [(rg2, 0.95 * rg2 + 0.05 * (g ** 2)) @@ -329,13 +334,13 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): zipped_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_grad' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] running_grads = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] running_grads2 = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_rgrad2' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] zgup = [(zg, g) for zg, g in zip(zipped_grads, grads)] rgup = [(rg, 0.95 * rg + 0.05 * g) for rg, g in zip(running_grads, grads)] @@ -348,7 +353,7 @@ def rmsprop(lr, tparams, grads, x, mask, y, cost): updir = [theano.shared(p.get_value() * numpy_floatX(0.), name='%s_updir' % k) - for k, p in tparams.iteritems()] + for k, p in tparams.items()] updir_new = [(ud, 0.9 * ud - 1e-4 * zg / tensor.sqrt(rg2 - rg ** 2 + 1e-4)) for ud, zg, rg, rg2 in zip(updir, zipped_grads, running_grads, running_grads2)] @@ -418,7 +423,7 @@ def pred_probs(f_pred_prob, prepare_data, data, iterator, verbose=False): n_done += len(valid_index) if verbose: - print '%d/%d samples classified' % (n_done, n_samples) + print('%d/%d samples classified' % (n_done, n_samples)) return probs @@ -470,11 +475,11 @@ def train_lstm( # Model options model_options = locals().copy() - print "model options", model_options + print("model options", model_options) load_data, prepare_data = get_dataset(dataset) - print 'Loading data' + print('Loading data') train, valid, test = load_data(n_words=n_words, valid_portion=0.05, maxlen=maxlen) if test_size > 0: @@ -490,7 +495,7 @@ def train_lstm( model_options['ydim'] = ydim - print 'Building model' + print('Building model') # This create the initial parameters as numpy ndarrays. # Dict name (string) -> numpy ndarray params = init_params(model_options) @@ -516,30 +521,30 @@ def train_lstm( f_cost = theano.function([x, mask, y], cost, name='f_cost') - grads = tensor.grad(cost, wrt=tparams.values()) + grads = tensor.grad(cost, wrt=list(tparams.values())) f_grad = theano.function([x, mask, y], grads, name='f_grad') lr = tensor.scalar(name='lr') f_grad_shared, f_update = optimizer(lr, tparams, grads, x, mask, y, cost) - print 'Optimization' + print('Optimization') kf_valid = get_minibatches_idx(len(valid[0]), valid_batch_size) kf_test = get_minibatches_idx(len(test[0]), valid_batch_size) - print "%d train examples" % len(train[0]) - print "%d valid examples" % len(valid[0]) - print "%d test examples" % len(test[0]) + print("%d train examples" % len(train[0])) + print("%d valid examples" % len(valid[0])) + print("%d test examples" % len(test[0])) history_errs = [] best_p = None bad_count = 0 if validFreq == -1: - validFreq = len(train[0]) / batch_size + validFreq = len(train[0]) // batch_size if saveFreq == -1: - saveFreq = len(train[0]) / batch_size + saveFreq = len(train[0]) // batch_size uidx = 0 # the number of update done estop = False # early stop @@ -569,22 +574,22 @@ def train_lstm( f_update(lrate) if numpy.isnan(cost) or numpy.isinf(cost): - print 'bad cost detected: ', cost + print('bad cost detected: ', cost) return 1., 1., 1. if numpy.mod(uidx, dispFreq) == 0: - print 'Epoch ', eidx, 'Update ', uidx, 'Cost ', cost + print('Epoch ', eidx, 'Update ', uidx, 'Cost ', cost) if saveto and numpy.mod(uidx, saveFreq) == 0: - print 'Saving...', + print('Saving...') if best_p is not None: params = best_p else: params = unzip(tparams) numpy.savez(saveto, history_errs=history_errs, **params) - pkl.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) - print 'Done' + pickle.dump(model_options, open('%s.pkl' % saveto, 'wb'), -1) + print('Done') if numpy.mod(uidx, validFreq) == 0: use_noise.set_value(0.) @@ -602,25 +607,25 @@ def train_lstm( best_p = unzip(tparams) bad_counter = 0 - print ('Train ', train_err, 'Valid ', valid_err, - 'Test ', test_err) + print( ('Train ', train_err, 'Valid ', valid_err, + 'Test ', test_err) ) if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, 0].min()): bad_counter += 1 if bad_counter > patience: - print 'Early Stop!' + print('Early Stop!') estop = True break - print 'Seen %d samples' % n_samples + print('Seen %d samples' % n_samples) if estop: break except KeyboardInterrupt: - print "Training interupted" + print("Training interupted") end_time = time.time() if best_p is not None: @@ -634,15 +639,15 @@ def train_lstm( valid_err = pred_error(f_pred, prepare_data, valid, kf_valid) test_err = pred_error(f_pred, prepare_data, test, kf_test) - print 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err + print( 'Train ', train_err, 'Valid ', valid_err, 'Test ', test_err ) if saveto: numpy.savez(saveto, train_err=train_err, valid_err=valid_err, test_err=test_err, history_errs=history_errs, **best_p) - print 'The code run for %d epochs, with %f sec/epochs' % ( - (eidx + 1), (end_time - start_time) / (1. * (eidx + 1))) - print >> sys.stderr, ('Training took %.1fs' % - (end_time - start_time)) + print('The code run for %d epochs, with %f sec/epochs' % ( + (eidx + 1), (end_time - start_time) / (1. * (eidx + 1)))) + print( ('Training took %.1fs' % + (end_time - start_time)), file=sys.stderr) return train_err, valid_err, test_err diff --git a/code/mlp.py b/code/mlp.py index 18f34e7c..1d463d81 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -18,6 +18,9 @@ Christopher M. Bishop, section 5 """ + +from __future__ import print_function + __docformat__ = 'restructedtext en' @@ -231,14 +234,14 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size + n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size + n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size ###################### # BUILD ACTUAL MODEL # ###################### - print '... building the model' + print('... building the model') # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch @@ -322,7 +325,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, ############### # TRAIN MODEL # ############### - print '... training' + print('... training') # early-stopping parameters patience = 10000 # look as this many examples regardless @@ -330,7 +333,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant - validation_frequency = min(n_train_batches, patience / 2) + validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we @@ -346,7 +349,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_model(minibatch_index) # iteration number @@ -355,7 +358,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, if (iter + 1) % validation_frequency == 0: # compute zero-one loss on validation set validation_losses = [validate_model(i) for i - in xrange(n_valid_batches)] + in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print( @@ -382,7 +385,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, # test it on the test set test_losses = [test_model(i) for i - in xrange(n_test_batches)] + in range(n_test_batches)] test_score = numpy.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of ' @@ -398,9 +401,9 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, print(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i, with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) - print >> sys.stderr, ('The code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) / 60.)) + print(('The code for file ' + + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.)), file=sys.stderr) if __name__ == '__main__': diff --git a/code/rbm.py b/code/rbm.py index 1ba4c86d..0a947963 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -4,6 +4,10 @@ contain hidden variables. Restricted Boltzmann Machines further restrict BMs to those without visible-visible and hidden-hidden connections. """ + +from __future__ import print_function +from six.moves import xrange + import timeit try: @@ -384,7 +388,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch @@ -438,7 +442,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, for batch_index in xrange(n_train_batches): mean_cost += [train_rbm(batch_index)] - print 'Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost) + print('Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)) # Plot filters after each training epoch plotting_start = timeit.default_timer() @@ -522,7 +526,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, # generate `plot_every` intermediate samples that we discard, # because successive samples in the chain are too correlated vis_mf, vis_sample = sample_fn() - print ' ... plotting sample ', idx + print(' ... plotting sample %d' % idx) image_data[29 * idx:29 * idx + 28, :] = tile_raster_images( X=vis_mf, img_shape=(28, 28), diff --git a/code/utils.py b/code/utils.py index 3b50019c..fa4e4d96 100644 --- a/code/utils.py +++ b/code/utils.py @@ -7,6 +7,7 @@ """ +from six.moves import xrange import numpy From 53f246dc2cd743131fd918542b7f24936e2decce Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Thu, 21 Jan 2016 16:02:02 -0500 Subject: [PATCH 248/417] partly fixed rnnrbm, but we will need to do some magic with the midi module to make it compatible with python 3 --- code/rnnrbm.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/code/rnnrbm.py b/code/rnnrbm.py index e1f40b5a..e5027083 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -3,6 +3,9 @@ # RNN-RBM deep learning tutorial # More information at http://deeplearning.net/tutorial/rnnrbm.html +from __future__ import print_function +from six.moves import xrange + import glob import os import sys @@ -11,10 +14,8 @@ try: import pylab except ImportError: - print ( - "pylab isn't available. If you use its functionality, it will crash." - ) - print "It can be installed with 'pip install -q Pillow'" + print ("pylab isn't available. If you use its functionality, it will crash.") + print("It can be installed with 'pip install -q Pillow'") from midi.utils import midiread, midiwrite import theano @@ -257,12 +258,12 @@ def train(self, files, batch_size=100, num_epochs=200): cost = self.train_function(sequence[i:i + batch_size]) costs.append(cost) - print 'Epoch %i/%i' % (epoch + 1, num_epochs), - print numpy.mean(costs) + print('Epoch %i/%i' % (epoch + 1, num_epochs)) + print(numpy.mean(costs)) sys.stdout.flush() except KeyboardInterrupt: - print 'Interrupted by user.' + print('Interrupted by user.') def generate(self, filename, show=True): '''Generate a sample sequence, plot the resulting piano-roll and save From 2c610d38168a38fbd0aa8fc032579114ff660cf2 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Thu, 28 Jan 2016 16:26:55 -0500 Subject: [PATCH 249/417] made rnnslu compatible with python 3. tested on cpu for many epochs, but not to completion --- code/rnnslu.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 2251e465..110029f4 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -1,6 +1,10 @@ + +from __future__ import print_function +from six.moves import xrange +import six.moves.cPickle as pickle + from collections import OrderedDict import copy -import cPickle import gzip import os import urllib @@ -66,7 +70,10 @@ def atisfold(fold): assert fold in range(5) filename = os.path.join(PREFIX, 'atis.fold'+str(fold)+'.pkl.gz') f = gzip.open(filename, 'rb') - train_set, valid_set, test_set, dicts = cPickle.load(f) + try: + train_set, valid_set, test_set, dicts = pickle.load(f, encoding='latin1') + except: + train_set, valid_set, test_set, dicts = pickle.load(f) return train_set, valid_set, test_set, dicts @@ -107,7 +114,7 @@ def download(origin, destination): download the corresponding atis file from http://www-etud.iro.umontreal.ca/~mesnilgr/atis/ ''' - print 'Downloading data from %s' % origin + print('Downloading data from %s' % origin) urllib.urlretrieve(origin, destination) @@ -125,8 +132,10 @@ def get_perf(filename, folder): stdin=subprocess.PIPE, stdout=subprocess.PIPE) - stdout, _ = proc.communicate(''.join(open(filename).readlines())) + stdout, _ = proc.communicate(''.join(open(filename).readlines()).encode('utf-8')) + stdout = stdout.decode('utf-8') out = None + for line in stdout.split('\n'): if 'accuracy' in line: out = line.split() @@ -237,7 +246,7 @@ def recurrence(x_t, h_tm1): def train(self, x, y, window_size, learning_rate): cwords = contextwin(x, window_size) - words = map(lambda x: numpy.asarray(x).astype('int32'), cwords) + words = list(map(lambda x: numpy.asarray(x).astype('int32'), cwords)) labels = y self.sentence_train(words, labels, learning_rate) @@ -274,7 +283,7 @@ def main(param=None): 'nepochs': 60, # 60 is recommended 'savemodel': False} - print param + print(param) folder_name = os.path.basename(__file__).split('.')[0] folder = os.path.join(os.path.dirname(__file__), folder_name) @@ -284,8 +293,8 @@ def main(param=None): # load the dataset train_set, valid_set, test_set, dic = atisfold(param['fold']) - idx2label = dict((k, v) for v, k in dic['labels2idx'].iteritems()) - idx2word = dict((k, v) for v, k in dic['words2idx'].iteritems()) + idx2label = dict((k, v) for v, k in dic['labels2idx'].items()) + idx2word = dict((k, v) for v, k in dic['words2idx'].items()) train_lex, train_ne, train_y = train_set valid_lex, valid_ne, valid_y = valid_set @@ -323,9 +332,9 @@ def main(param=None): for i, (x, y) in enumerate(zip(train_lex, train_y)): rnn.train(x, y, param['win'], param['clr']) - print '[learning] epoch %i >> %2.2f%%' % ( - e, (i + 1) * 100. / nsentences), - print 'completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic), + print('[learning] epoch %i >> %2.2f%%' % ( + e, (i + 1) * 100. / nsentences),) + print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic),) sys.stdout.flush() # evaluation // back into the real world : idx -> words @@ -374,7 +383,7 @@ def main(param=None): folder + '/best.valid.txt']) else: if param['verbose']: - print '' + print('') # learning rate decay if no improvement in 10 epochs if param['decay'] and abs(param['be']-param['ce']) >= 10: @@ -384,10 +393,10 @@ def main(param=None): if param['clr'] < 1e-5: break - print('BEST RESULT: epoch', param['be'], - 'valid F1', param['vf1'], - 'best test F1', param['tf1'], - 'with the model', folder) + print(('BEST RESULT: epoch', param['be'], + 'valid F1', param['vf1'], + 'best test F1', param['tf1'], + 'with the model', folder)) if __name__ == '__main__': From 226729f96785a96b22a937de199abab62e830de4 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Fri, 29 Jan 2016 15:55:43 -0500 Subject: [PATCH 250/417] all fixes suggested by Pascal, plus update for the doc --- code/SdA.py | 3 +-- code/lstm.py | 1 - code/rnnslu.py | 8 ++++---- doc/index.txt | 3 +++ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index d639cb54..25e306c7 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -394,8 +394,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) - print('Pre-training layer %i, epoch %d, cost ' % (i, epoch)) - print(numpy.mean(c)) + print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, numpy.mean(c))) end_time = timeit.default_timer() diff --git a/code/lstm.py b/code/lstm.py index b3b89f3e..9c19e1ad 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -6,7 +6,6 @@ from six.moves import xrange import six.moves.cPickle as pickle -#from six.moves.collections import OrderedDict from collections import OrderedDict import sys import time diff --git a/code/rnnslu.py b/code/rnnslu.py index 110029f4..45aaf3a6 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -333,8 +333,8 @@ def main(param=None): for i, (x, y) in enumerate(zip(train_lex, train_y)): rnn.train(x, y, param['win'], param['clr']) print('[learning] epoch %i >> %2.2f%%' % ( - e, (i + 1) * 100. / nsentences),) - print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic),) + e, (i + 1) * 100. / nsentences), end=' ') + print('completed in %.2f (sec) <<\r' % (timeit.default_timer() - tic), end='') sys.stdout.flush() # evaluation // back into the real world : idx -> words @@ -393,10 +393,10 @@ def main(param=None): if param['clr'] < 1e-5: break - print(('BEST RESULT: epoch', param['be'], + print('BEST RESULT: epoch', param['be'], 'valid F1', param['vf1'], 'best test F1', param['tf1'], - 'with the model', folder)) + 'with the model', folder) if __name__ == '__main__': diff --git a/doc/index.txt b/doc/index.txt index 7c6605bf..68a18ec5 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -63,3 +63,6 @@ Energy-based recurrent neural network (RNN-RBM): .. _Theano basic tutorial: http://deeplearning.net/software/theano/tutorial .. _Contractive auto-encoders: https://github.com/lisa-lab/DeepLearningTutorials/blob/master/code/cA.py + +Note that the tutorials here are all compatible with Python 2 and 3, with the exception of :ref:`rnnrbm` which is only available for Python 2. + From 90b925b2c716f29b26209375fc28b1e32fad6f22 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Mon, 1 Feb 2016 13:36:43 -0500 Subject: [PATCH 251/417] travis python version update as suggested by Fred --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 258963ee..e2f2d530 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ sudo: false language: python #python: -# - "2.7" -# - "3.2" +# - "2.6" +# - "3.3" # command to install dependencies before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh From 4c0858de1073660842f3f9b8f53c162ca3107653 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Mon, 1 Feb 2016 15:51:10 -0500 Subject: [PATCH 252/417] got rid of all the xrange --- code/DBN.py | 14 +++++++------- code/cA.py | 5 ++--- code/hmc/test_hmc.py | 5 ++--- code/logistic_cg.py | 8 ++++---- code/lstm.py | 3 +-- code/rbm.py | 7 +++---- code/rnnrbm.py | 5 ++--- code/rnnslu.py | 3 +-- code/utils.py | 8 +++----- doc/gettingstarted.txt | 2 +- doc/utilities.txt | 6 +++--- 11 files changed, 29 insertions(+), 37 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index ecd563e7..6ca88603 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -75,7 +75,7 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, # training the DBN by doing stochastic gradient descent on the # MLP. - for i in xrange(self.n_layers): + for i in range(self.n_layers): # construct the sigmoidal layer # the size of the input is either the number of hidden @@ -267,11 +267,11 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): # Create a function that scans the entire validation set def valid_score(): - return [valid_score_i(i) for i in xrange(n_valid_batches)] + return [valid_score_i(i) for i in range(n_valid_batches)] # Create a function that scans the entire test set def test_score(): - return [test_score_i(i) for i in xrange(n_test_batches)] + return [test_score_i(i) for i in range(n_test_batches)] return train_fn, valid_score, test_score @@ -329,12 +329,12 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, print '... pre-training the model' start_time = timeit.default_timer() ## Pre-train layer-wise - for i in xrange(dbn.n_layers): + for i in range(dbn.n_layers): # go through pretraining epochs - for epoch in xrange(pretraining_epochs): + for epoch in range(pretraining_epochs): # go through the training set c = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr)) print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), @@ -379,7 +379,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, while (epoch < training_epochs) and (not done_looping): epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index diff --git a/code/cA.py b/code/cA.py index 0d563ef2..8dc5d8b6 100644 --- a/code/cA.py +++ b/code/cA.py @@ -30,7 +30,6 @@ """ from __future__ import print_function -from six.moves import xrange import os import sys @@ -287,10 +286,10 @@ def test_cA(learning_rate=0.01, training_epochs=20, ############ # go through training epochs - for epoch in xrange(training_epochs): + for epoch in range(training_epochs): # go through trainng set c = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): c.append(train_ca(batch_index)) c_array = numpy.vstack(c) diff --git a/code/hmc/test_hmc.py b/code/hmc/test_hmc.py index c3c425e6..be1a1ac6 100644 --- a/code/hmc/test_hmc.py +++ b/code/hmc/test_hmc.py @@ -1,6 +1,5 @@ from __future__ import print_function -from six.moves import xrange import numpy import theano @@ -39,10 +38,10 @@ def gaussian_energy(x): initial_stepsize=1e-3, stepsize_max=0.5) # Start with a burn-in process - garbage = [sampler.draw() for r in xrange(burnin)] # burn-in Draw + garbage = [sampler.draw() for r in range(burnin)] # burn-in Draw # `n_samples`: result is a 3D tensor of dim [n_samples, batchsize, # dim] - _samples = numpy.asarray([sampler.draw() for r in xrange(n_samples)]) + _samples = numpy.asarray([sampler.draw() for r in range(n_samples)]) # Flatten to [n_samples * batchsize, dim] samples = _samples.T.reshape(dim, -1).T diff --git a/code/logistic_cg.py b/code/logistic_cg.py index db9822ef..40c72c2f 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -239,7 +239,7 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): def train_fn(theta_value): classifier.theta.set_value(theta_value, borrow=True) train_losses = [batch_cost(i * batch_size) - for i in xrange(n_train_batches)] + for i in range(n_train_batches)] return numpy.mean(train_losses) # creates a function that computes the average gradient of cost with @@ -247,7 +247,7 @@ def train_fn(theta_value): def train_fn_grad(theta_value): classifier.theta.set_value(theta_value, borrow=True) grad = batch_grad(0) - for i in xrange(1, n_train_batches): + for i in range(1, n_train_batches): grad += batch_grad(i * batch_size) return grad / n_train_batches @@ -258,7 +258,7 @@ def callback(theta_value): classifier.theta.set_value(theta_value, borrow=True) #compute the validation loss validation_losses = [validate_model(i * batch_size) - for i in xrange(n_valid_batches)] + for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print('validation error %f %%' % (this_validation_loss * 100.,)) @@ -268,7 +268,7 @@ def callback(theta_value): # testing dataset validation_scores[0] = this_validation_loss test_losses = [test_model(i * batch_size) - for i in xrange(n_test_batches)] + for i in range(n_test_batches)] validation_scores[1] = numpy.mean(test_losses) ############### diff --git a/code/lstm.py b/code/lstm.py index 9c19e1ad..1c285928 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -3,7 +3,6 @@ ''' from __future__ import print_function -from six.moves import xrange import six.moves.cPickle as pickle from collections import OrderedDict @@ -549,7 +548,7 @@ def train_lstm( estop = False # early stop start_time = time.time() try: - for eidx in xrange(max_epochs): + for eidx in range(max_epochs): n_samples = 0 # Get new shuffled index for the training set. diff --git a/code/rbm.py b/code/rbm.py index 0a947963..901b5870 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -6,7 +6,6 @@ """ from __future__ import print_function -from six.moves import xrange import timeit @@ -435,11 +434,11 @@ def test_rbm(learning_rate=0.1, training_epochs=15, start_time = timeit.default_timer() # go through training epochs - for epoch in xrange(training_epochs): + for epoch in range(training_epochs): # go through the training set mean_cost = [] - for batch_index in xrange(n_train_batches): + for batch_index in range(n_train_batches): mean_cost += [train_rbm(batch_index)] print('Training epoch %d, cost is ' % epoch, numpy.mean(mean_cost)) @@ -522,7 +521,7 @@ def test_rbm(learning_rate=0.1, training_epochs=15, (29 * n_samples + 1, 29 * n_chains - 1), dtype='uint8' ) - for idx in xrange(n_samples): + for idx in range(n_samples): # generate `plot_every` intermediate samples that we discard, # because successive samples in the chain are too correlated vis_mf, vis_sample = sample_fn() diff --git a/code/rnnrbm.py b/code/rnnrbm.py index e5027083..b8420b9b 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -4,7 +4,6 @@ # More information at http://deeplearning.net/tutorial/rnnrbm.html from __future__ import print_function -from six.moves import xrange import glob import os @@ -249,12 +248,12 @@ def train(self, files, batch_size=100, num_epochs=200): for f in files] try: - for epoch in xrange(num_epochs): + for epoch in range(num_epochs): numpy.random.shuffle(dataset) costs = [] for s, sequence in enumerate(dataset): - for i in xrange(0, len(sequence), batch_size): + for i in range(0, len(sequence), batch_size): cost = self.train_function(sequence[i:i + batch_size]) costs.append(cost) diff --git a/code/rnnslu.py b/code/rnnslu.py index 45aaf3a6..0413ee63 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -1,6 +1,5 @@ from __future__ import print_function -from six.moves import xrange import six.moves.cPickle as pickle from collections import OrderedDict @@ -322,7 +321,7 @@ def main(param=None): # train with early stopping on validation set best_f1 = -numpy.inf param['clr'] = param['lr'] - for e in xrange(param['nepochs']): + for e in range(param['nepochs']): # shuffle shuffle([train_lex, train_ne, train_y], param['seed']) diff --git a/code/utils.py b/code/utils.py index fa4e4d96..ff772ad4 100644 --- a/code/utils.py +++ b/code/utils.py @@ -6,8 +6,6 @@ image from a set of samples or weights. """ - -from six.moves import xrange import numpy @@ -86,7 +84,7 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), else: channel_defaults = [0., 0., 0., 1.] - for i in xrange(4): + for i in range(4): if X[i] is None: # if channel is None, fill it with zeros of the correct # dtype @@ -116,8 +114,8 @@ def tile_raster_images(X, img_shape, tile_shape, tile_spacing=(0, 0), dt = 'uint8' out_array = numpy.zeros(out_shape, dtype=dt) - for tile_row in xrange(tile_shape[0]): - for tile_col in xrange(tile_shape[1]): + for tile_row in range(tile_shape[0]): + for tile_col in range(tile_shape[1]): if tile_row * tile_shape[1] + tile_col < X.shape[0]: this_x = X[tile_row * tile_shape[1] + tile_col] if scale_rows_to_unit_interval: diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index e838d706..d765f14a 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -578,7 +578,7 @@ of a strategy based on a geometrically increasing amount of patience. while (epoch < n_epochs) and (not done_looping): # Report "1" for first epoch, "n_epochs" for last epoch epoch = epoch + 1 - for minibatch_index in xrange(n_train_batches): + for minibatch_index in range(n_train_batches): d_loss_wrt_params = ... # compute gradient params -= learning_rate * d_loss_wrt_params # gradient descent diff --git a/doc/utilities.txt b/doc/utilities.txt index 0367127c..eb982ec2 100644 --- a/doc/utilities.txt +++ b/doc/utilities.txt @@ -112,7 +112,7 @@ Tiling minibatches together is done for us by the else: channel_defaults = [0., 0., 0., 1.] - for i in xrange(4): + for i in range(4): if X[i] is None: # if channel is None, fill it with zeros of the correct # dtype @@ -134,8 +134,8 @@ Tiling minibatches together is done for us by the out_array = numpy.zeros(out_shape, dtype='uint8' if output_pixel_vals else X.dtype) - for tile_row in xrange(tile_shape[0]): - for tile_col in xrange(tile_shape[1]): + for tile_row in range(tile_shape[0]): + for tile_col in range(tile_shape[1]): if tile_row * tile_shape[1] + tile_col < X.shape[0]: if scale_rows_to_unit_interval: # if we should scale values to be between 0 and 1 From dcfe518dba2e346268ac88884578db5ce4fbebf4 Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Wed, 3 Feb 2016 16:37:27 -0500 Subject: [PATCH 253/417] minor edit to respond to Pascal's suggestion --- code/hmc/test_hmc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/hmc/test_hmc.py b/code/hmc/test_hmc.py index be1a1ac6..42dbc3a7 100644 --- a/code/hmc/test_hmc.py +++ b/code/hmc/test_hmc.py @@ -6,7 +6,7 @@ try: from hmc import HMC_sampler -except: +except ImportError as e: # python 3 compatibility # http://stackoverflow.com/questions/3073259/python-nose-import-error from hmc.hmc import HMC_sampler From 8ca9239cbd9ad4472241bad638c4b283818295da Mon Sep 17 00:00:00 2001 From: Guillaume Alain Date: Tue, 9 Feb 2016 11:31:59 -0500 Subject: [PATCH 254/417] missed one print statement --- code/rnnslu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/rnnslu.py b/code/rnnslu.py index 0413ee63..3c620178 100644 --- a/code/rnnslu.py +++ b/code/rnnslu.py @@ -141,7 +141,7 @@ def get_perf(filename, folder): break # To help debug if out is None: - print stdout.split('\n') + print(stdout.split('\n')) precision = float(out[6][:-2]) recall = float(out[8][:-2]) f1score = float(out[10]) From 0054116a1cadc27fe6353f14ee48479e681c0b19 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 17 Feb 2016 09:02:43 -0500 Subject: [PATCH 255/417] Update timing due to speed up. (lower the number of random number generator) --- code/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/code/test.py b/code/test.py index 39d0ab4c..b08f39a3 100644 --- a/code/test.py +++ b/code/test.py @@ -103,9 +103,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 510.9, 130.4, 23.2, 98.8]) + 346.9, 355.0, 268.2, 130.4, 23.2, 98.8]) expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 66.5, 71, - 191.2, 199.0, 400.4, 119.5, 36.9, 67.2]) + 191.2, 199.0, 201.9, 119.5, 36.9, 67.2]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -125,8 +125,8 @@ def speed(): # 1.35324519 1.7356905 1.12937868] expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, - 5.8, 20.0, - 11.2, 17.2, 244.3, 118.8, 34.2, 8.7]) + 5.8, 19.2, + 11.2, 17.2, 122, 112.5, 31.1, 8.7]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 0ef0b4dd4d9ebcacb21c8079595637bc1742e588 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 17 Feb 2016 10:24:53 -0500 Subject: [PATCH 256/417] Make DLT compatible with Theano 0.7 --- code/DBN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/DBN.py b/code/DBN.py index 6ca88603..b8e35fad 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -174,7 +174,7 @@ def pretraining_functions(self, train_set_x, batch_size, k): # compile the theano function fn = theano.function( - inputs=[index, theano.In(learning_rate, value=0.1)], + inputs=[index, theano.Param(learning_rate, default=0.1)], outputs=cost, updates=updates, givens={ From 0c8507bc469e0a99027350c526372b8c8dd8a75d Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 24 Feb 2016 16:29:05 -0500 Subject: [PATCH 257/417] Update speed test to faster speed --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index b08f39a3..250e4d7e 100644 --- a/code/test.py +++ b/code/test.py @@ -104,7 +104,7 @@ def speed(): expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, 346.9, 355.0, 268.2, 130.4, 23.2, 98.8]) - expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 66.5, 71, + expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71, 191.2, 199.0, 201.9, 119.5, 36.9, 67.2]) # Number with just 1 decimal are new value that are faster with From cdfcde08e4667d794db3907ae19437c352baab85 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 2 Mar 2016 09:13:43 -0500 Subject: [PATCH 258/417] Speed up 8 expected benchmark speed --- code/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/test.py b/code/test.py index 250e4d7e..e034c63b 100644 --- a/code/test.py +++ b/code/test.py @@ -103,9 +103,9 @@ def speed(): # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 268.2, 130.4, 23.2, 98.8]) + 346.9, 355.0, 268.2, 115.8, 16.8, 91.6]) expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71, - 191.2, 199.0, 201.9, 119.5, 36.9, 67.2]) + 191.2, 199.0, 201.9, 107, 12.6, 61.3]) # Number with just 1 decimal are new value that are faster with # the Theano version 0.5rc2 Other number are older. They are not @@ -126,7 +126,7 @@ def speed(): expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, 5.8, 19.2, - 11.2, 17.2, 122, 112.5, 31.1, 8.7]) + 11.2, 7.8, 122, 112.5, 31.1, 8.3]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From bba82fbe92447b7e346a941847581199c05e4eeb Mon Sep 17 00:00:00 2001 From: Jamie White Date: Wed, 9 Mar 2016 22:12:06 -0500 Subject: [PATCH 259/417] Update mlp.py Fixed misspelling of "sorted" --- code/mlp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/mlp.py b/code/mlp.py index 1d463d81..e865bc8f 100644 --- a/code/mlp.py +++ b/code/mlp.py @@ -292,7 +292,7 @@ def test_mlp(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, ) # start-snippet-5 - # compute the gradient of cost with respect to theta (sotred in params) + # compute the gradient of cost with respect to theta (sorted in params) # the resulting gradients will be stored in a list gparams gparams = [T.grad(cost, param) for param in classifier.params] From 06a9d877642ed22ceccaf913edfb746a013e9184 Mon Sep 17 00:00:00 2001 From: Kyunghyun Cho Date: Fri, 18 Mar 2016 10:15:24 -0400 Subject: [PATCH 260/417] no nonlinearity in z --- doc/lstm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index 828fd694..bde70bd8 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -174,7 +174,7 @@ be computed with : .. math:: - z = \sigma(W x_t + U h_{t-1} + b) + z = W x_t + U h_{t-1} + b The result is then sliced to obtain the pre-nonlinearity activations for :math:`i`, :math:`f`, :math:`\widetilde{C_t}`, and :math:`o` and the From 146eb2a3680658cca971d2aa3c3f1ab1471075b0 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 9 Mar 2016 09:27:38 -0500 Subject: [PATCH 261/417] Don't be too much version when downloading. Make buildbot output smaller --- data/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/download.sh b/data/download.sh index ed273bbb..160b0986 100755 --- a/data/download.sh +++ b/data/download.sh @@ -5,7 +5,7 @@ WGET=$? which curl >/dev/null 2>&1 CURL=$? if [ "$WGET" -eq 0 ]; then - DL_CMD="wget -c" + DL_CMD="wget --no-verbose -c" elif [ "$CURL" -eq 0 ]; then DL_CMD="curl -C - -O" else From 57a80fd2bb51b171b81db05cbd33bcfaf68e322f Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 9 Mar 2016 09:27:59 -0500 Subject: [PATCH 262/417] Give name to theano function --- code/rbm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/code/rbm.py b/code/rbm.py index 901b5870..3800cca7 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -257,7 +257,8 @@ def get_cost_updates(self, lr=0.1, persistent=None, k=1): # chain_start is the initial state corresponding to the # 6th output outputs_info=[None, None, None, None, None, chain_start], - n_steps=k + n_steps=k, + name="gibbs_hvh" ) # start-snippet-3 # determine gradients on RBM parameters @@ -496,7 +497,8 @@ def test_rbm(learning_rate=0.1, training_epochs=15, ) = theano.scan( rbm.gibbs_vhv, outputs_info=[None, None, None, None, None, persistent_vis_chain], - n_steps=plot_every + n_steps=plot_every, + name="gibbs_vhv" ) # add to updates the shared variable that takes care of our persistent From ff6939b7bcdb70c7acbd9ed4020eacbb0a65c6d0 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Tue, 22 Mar 2016 11:05:15 -0400 Subject: [PATCH 263/417] Finish passing to new conv2d interface --- code/convolutional_mlp.py | 2 +- doc/lenet.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index a8811bc1..62845c99 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -94,7 +94,7 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): input=input, filters=self.W, filter_shape=filter_shape, - image_shape=image_shape + input_shape=image_shape ) # downsample each feature map individually, using maxpooling diff --git a/doc/lenet.txt b/doc/lenet.txt index 117dfdab..76614106 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -196,7 +196,7 @@ one of Figure 1. The input consists of 3 features maps (an RGB color image) of s import theano from theano import tensor as T - from theano.tensor.nnet import conv + from theano.tensor.nnet import conv2d import numpy @@ -226,7 +226,7 @@ one of Figure 1. The input consists of 3 features maps (an RGB color image) of s dtype=input.dtype), name ='b') # build symbolic expression that computes the convolution of input with filters in w - conv_out = conv.conv2d(input, W) + conv_out = conv2d(input, W) # build symbolic expression to add bias and apply activation function, i.e. produce neural net layer output # A few words on ``dimshuffle`` : @@ -404,7 +404,7 @@ to be compatible with our previous MLP implementation. Note that the term "convolution" could corresponds to different mathematical operations: 1. `theano.tensor.nnet.conv2d - `_, + `_, which is the most common one in almost all of the recent published convolutional models. In this operation, each output feature map is connected to each From ee5c0cb9a5e873d51c25dc60203e828dd1793889 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Tue, 22 Mar 2016 11:07:09 -0400 Subject: [PATCH 264/417] Use the new Interface --- code/DBN.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/DBN.py b/code/DBN.py index b8e35fad..6ca88603 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -174,7 +174,7 @@ def pretraining_functions(self, train_set_x, batch_size, k): # compile the theano function fn = theano.function( - inputs=[index, theano.Param(learning_rate, default=0.1)], + inputs=[index, theano.In(learning_rate, value=0.1)], outputs=cost, updates=updates, givens={ From 797342acc73b94854964e682ec5babbc5735bdfc Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 1 Apr 2016 13:34:19 -0400 Subject: [PATCH 265/417] Use the new pool interface --- code/convolutional_mlp.py | 6 +++--- doc/lenet.txt | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/code/convolutional_mlp.py b/code/convolutional_mlp.py index 62845c99..6bbb47a1 100644 --- a/code/convolutional_mlp.py +++ b/code/convolutional_mlp.py @@ -32,7 +32,7 @@ import theano import theano.tensor as T -from theano.tensor.signal import downsample +from theano.tensor.signal import pool from theano.tensor.nnet import conv2d from logistic_sgd import LogisticRegression, load_data @@ -97,8 +97,8 @@ def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): input_shape=image_shape ) - # downsample each feature map individually, using maxpooling - pooled_out = downsample.max_pool_2d( + # pool each feature map individually, using maxpooling + pooled_out = pool.pool_2d( input=conv_out, ds=poolsize, ignore_border=True diff --git a/doc/lenet.txt b/doc/lenet.txt index 76614106..09f50be6 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -7,7 +7,7 @@ Convolutional Neural Networks (LeNet) This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally, it uses the following new Theano functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, - `floatX`_, `downsample`_ , `conv2d`_, `dimshuffle`_. If you intend to run the + `floatX`_, `pool`_ , `conv2d`_, `dimshuffle`_. If you intend to run the code on GPU also read `GPU`_. To run this example on a GPU, you need a good GPU. It needs @@ -35,7 +35,7 @@ Convolutional Neural Networks (LeNet) .. _GPU: http://deeplearning.net/software/theano/tutorial/using_gpu.html -.. _downsample: http://deeplearning.net/software/theano/library/tensor/signal/downsample.html +.. _pool: http://deeplearning.net/software/theano/library/tensor/signal/pool.html .. _conv2d: http://deeplearning.net/software/theano/library/tensor/signal/conv.html#module-conv @@ -320,7 +320,7 @@ Max-pooling is useful in vision for two reasons: "smart" way of reducing the dimensionality of intermediate representations. Max-pooling is done in Theano by way of -``theano.tensor.signal.downsample.max_pool_2d``. This function takes as input +``theano.tensor.signal.pool.pool_2d``. This function takes as input an N dimensional tensor (where N >= 2) and a downscaling factor and performs max-pooling over the 2 trailing dimensions of the tensor. @@ -328,11 +328,11 @@ An example is worth a thousand words: .. code-block:: python - from theano.tensor.signal import downsample + from theano.tensor.signal import pool input = T.dtensor4('input') maxpool_shape = (2, 2) - pool_out = downsample.max_pool_2d(input, maxpool_shape, ignore_border=True) + pool_out = pool.pool_2d(input, maxpool_shape, ignore_border=True) f = theano.function([input],pool_out) invals = numpy.random.RandomState(1).rand(3, 2, 5, 5) @@ -340,7 +340,7 @@ An example is worth a thousand words: print 'invals[0, 0, :, :] =\n', invals[0, 0, :, :] print 'output[0, 0, :, :] =\n', f(invals)[0, 0, :, :] - pool_out = downsample.max_pool_2d(input, maxpool_shape, ignore_border=False) + pool_out = pool.pool_2d(input, maxpool_shape, ignore_border=False) f = theano.function([input],pool_out) print 'With ignore_border set to False:' print 'invals[1, 0, :, :] =\n ', invals[1, 0, :, :] From aad4f16662edb643926a38b661f469b6026a6a72 Mon Sep 17 00:00:00 2001 From: "lorenzo.ritter" Date: Wed, 27 Apr 2016 19:10:25 +0200 Subject: [PATCH 266/417] fixed typo in SdA.py --- code/SdA.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index 25e306c7..3d9589ac 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -81,8 +81,8 @@ def __init__( :type n_ins: int :param n_ins: dimension of the input to the sdA - :type n_layers_sizes: list of ints - :param n_layers_sizes: intermediate layers size, must contain + :type hidden_layers_sizes: list of ints + :param hidden_layers_sizes: intermediate layers size, must contain at least one value :type n_outs: int From de99c6eb17d802549bf08fc7ed5ed4f287f967c2 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Sun, 8 May 2016 19:50:03 -0400 Subject: [PATCH 267/417] Commit a small speed up. --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index e034c63b..6aee1084 100644 --- a/code/test.py +++ b/code/test.py @@ -126,7 +126,7 @@ def speed(): expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, 5.8, 19.2, - 11.2, 7.8, 122, 112.5, 31.1, 8.3]) + 11.2, 7.3, 122, 112.5, 31.1, 8.3]) expected_times_64 = [s for idx, s in enumerate(expected_times_64) if to_exec[idx]] expected_times_32 = [s for idx, s in enumerate(expected_times_32) From 75cbba67b4fdc271bae5b7020a2a3fc69b70328d Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Wed, 13 Jul 2016 14:03:47 -0400 Subject: [PATCH 268/417] Python 3 + flake8 fixes. --- code/DBN.py | 101 +++++++++++++++++++--------------------- code/imdb_preprocess.py | 10 ++-- code/logistic_cg.py | 25 +++++----- code/test.py | 11 +++-- 4 files changed, 70 insertions(+), 77 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 6ca88603..3b2bd230 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -1,5 +1,6 @@ """ """ +from __future__ import print_function, division import os import sys import timeit @@ -61,9 +62,12 @@ def __init__(self, numpy_rng, theano_rng=None, n_ins=784, theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30)) # allocate symbolic variables for the data - self.x = T.matrix('x') # the data is presented as rasterized images - self.y = T.ivector('y') # the labels are presented as 1D vector - # of [int] labels + + # the data is presented as rasterized images + self.x = T.matrix('x') + + # the labels are presented as 1D vector of [int] labels + self.y = T.ivector('y') # end-snippet-1 # The DBN is an MLP, for which all weights of intermediate # layers are shared with a different RBM. We will first @@ -156,8 +160,6 @@ def pretraining_functions(self, train_set_x, batch_size, k): index = T.lscalar('index') # index to a minibatch learning_rate = T.scalar('lr') # learning rate to use - # number of batches - n_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size # begining of a batch, given `index` batch_begin = index * batch_size # ending of a batch given `index` @@ -211,9 +213,9 @@ def build_finetune_functions(self, datasets, batch_size, learning_rate): # compute number of minibatches for training, validation and testing n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] - n_valid_batches /= batch_size + n_valid_batches //= batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] - n_test_batches /= batch_size + n_test_batches //= batch_size index = T.lscalar('index') # index to a [mini]batch @@ -307,11 +309,11 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, test_set_x, test_set_y = datasets[2] # compute number of minibatches for training, validation and testing - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size # numpy random generator numpy_rng = numpy.random.RandomState(123) - print '... building the model' + print('... building the model') # construct the Deep Belief Network dbn = DBN(numpy_rng=numpy_rng, n_ins=28 * 28, hidden_layers_sizes=[1000, 1000, 1000], @@ -321,14 +323,14 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, ######################### # PRETRAINING THE MODEL # ######################### - print '... getting the pretraining functions' + print('... getting the pretraining functions') pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x, batch_size=batch_size, k=k) - print '... pre-training the model' + print('... pre-training the model') start_time = timeit.default_timer() - ## Pre-train layer-wise + # Pre-train layer-wise for i in range(dbn.n_layers): # go through pretraining epochs for epoch in range(pretraining_epochs): @@ -337,38 +339,40 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, for batch_index in range(n_train_batches): c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr)) - print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch), - print numpy.mean(c) + print('Pre-training layer %i, epoch %d, cost ' % (i, epoch), end=' ') + print(numpy.mean(c)) end_time = timeit.default_timer() # end-snippet-2 - print >> sys.stderr, ('The pretraining code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) / 60.)) + print('The pretraining code for file ' + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.), file=sys.stderr) ######################## # FINETUNING THE MODEL # ######################## # get the training, validation and testing function for the model - print '... getting the finetuning functions' + print('... getting the finetuning functions') train_fn, validate_model, test_model = dbn.build_finetune_functions( datasets=datasets, batch_size=batch_size, learning_rate=finetune_lr ) - print '... finetuning the model' + print('... finetuning the model') # early-stopping parameters - patience = 4 * n_train_batches # look as this many examples regardless - patience_increase = 2. # wait this much longer when a new best is - # found - improvement_threshold = 0.995 # a relative improvement of this much is - # considered significant + + # look as this many examples regardless + patience = 4 * n_train_batches + + # wait this much longer when a new best is found + patience_increase = 2. + + # a relative improvement of this much is considered significant + improvement_threshold = 0.995 + + # go through this many minibatches before checking the network on + # the validation set; in this case we check every epoch validation_frequency = min(n_train_batches, patience / 2) - # go through this many - # minibatches before checking the network - # on the validation set; in this case we - # check every epoch best_validation_loss = numpy.inf test_score = 0. @@ -381,31 +385,27 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, epoch = epoch + 1 for minibatch_index in range(n_train_batches): - minibatch_avg_cost = train_fn(minibatch_index) + train_fn(minibatch_index) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() this_validation_loss = numpy.mean(validation_losses) - print( - 'epoch %i, minibatch %i/%i, validation error %f %%' - % ( - epoch, - minibatch_index + 1, - n_train_batches, - this_validation_loss * 100. + print('epoch %i, minibatch %i/%i, validation error %f %%' % ( + epoch, + minibatch_index + 1, + n_train_batches, + this_validation_loss * 100. ) ) # if we got the best validation score until now if this_validation_loss < best_validation_loss: - #improve patience if loss improvement is good enough - if ( - this_validation_loss < best_validation_loss * - improvement_threshold - ): + # improve patience if loss improvement is good enough + if (this_validation_loss < best_validation_loss * + improvement_threshold): patience = max(patience, iter * patience_increase) # save best validation score and iteration number @@ -418,24 +418,19 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, - test_score * 100.)) + test_score * 100.)) if patience <= iter: done_looping = True break end_time = timeit.default_timer() - print( - ( - 'Optimization complete with best validation score of %f %%, ' - 'obtained at iteration %i, ' - 'with test performance %f %%' - ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.) - ) - print >> sys.stderr, ('The fine tuning code for file ' + - os.path.split(__file__)[1] + - ' ran for %.2fm' % ((end_time - start_time) - / 60.)) + print(('Optimization complete with best validation score of %f %%, ' + 'obtained at iteration %i, ' + 'with test performance %f %%' + ) % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) + print('The fine tuning code for file ' + os.path.split(__file__)[1] + + ' ran for %.2fm' % ((end_time - start_time) / 60.), file=sys.stderr) if __name__ == '__main__': diff --git a/code/imdb_preprocess.py b/code/imdb_preprocess.py index c20b37b6..62ebb556 100644 --- a/code/imdb_preprocess.py +++ b/code/imdb_preprocess.py @@ -8,7 +8,7 @@ 3) Then run this script. """ - +from __future__ import print_function dataset_path='/Tmp/bastienf/aclImdb/' import numpy @@ -27,12 +27,12 @@ def tokenize(sentences): - print 'Tokenizing..', + print('Tokenizing..', end=' ') text = "\n".join(sentences) tokenizer = Popen(tokenizer_cmd, stdin=PIPE, stdout=PIPE) tok_text, _ = tokenizer.communicate(text) toks = tok_text.split('\n')[:-1] - print 'Done' + print('Done') return toks @@ -52,7 +52,7 @@ def build_dict(path): sentences = tokenize(sentences) - print 'Building dictionary..', + print('Building dictionary..', end=' ') wordcount = dict() for ss in sentences: words = ss.strip().lower().split() @@ -72,7 +72,7 @@ def build_dict(path): for idx, ss in enumerate(sorted_idx): worddict[keys[ss]] = idx+2 # leave 0 and 1 (UNK) - print numpy.sum(counts), ' total words ', len(keys), ' unique words' + print(numpy.sum(counts), ' total words ', len(keys), ' unique words') return worddict diff --git a/code/logistic_cg.py b/code/logistic_cg.py index 40c72c2f..c2970d51 100644 --- a/code/logistic_cg.py +++ b/code/logistic_cg.py @@ -33,6 +33,7 @@ """ +from __future__ import print_function, division __docformat__ = 'restructedtext en' @@ -165,9 +166,9 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): batch_size = 600 # size of the minibatch - n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size - n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size - n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size + n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size + n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size + n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size n_in = 28 * 28 # number of input units n_out = 10 # number of output units @@ -175,7 +176,7 @@ def cg_optimization_mnist(n_epochs=50, mnist_pkl_gz='mnist.pkl.gz'): ###################### # BUILD ACTUAL MODEL # ###################### - print '... building the model' + print('... building the model') # allocate symbolic variables for the data minibatch_offset = T.lscalar() # offset to the start of a [mini]batch @@ -260,7 +261,7 @@ def callback(theta_value): validation_losses = [validate_model(i * batch_size) for i in range(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) - print('validation error %f %%' % (this_validation_loss * 100.,)) + print(('validation error %f %%' % (this_validation_loss * 100.,))) # check if it is better then best validation score got until now if this_validation_loss < validation_scores[0]: @@ -288,17 +289,13 @@ def callback(theta_value): maxiter=n_epochs ) end_time = timeit.default_timer() - print( - ( - 'Optimization complete with best validation score of %f %%, with ' - 'test performance %f %%' - ) - % (validation_scores[0] * 100., validation_scores[1] * 100.) + print(('Optimization complete with best validation score of %f %%, with ' + 'test performance %f %%' + ) % (validation_scores[0] * 100., validation_scores[1] * 100.) ) - print >> sys.stderr, ('The code for file ' + - os.path.split(__file__)[1] + - ' ran for %.1fs' % ((end_time - start_time))) + print('The code for file ' + os.path.split(__file__)[1] + + ' ran for %.1fs' % (end_time - start_time), file=sys.stderr) if __name__ == '__main__': diff --git a/code/test.py b/code/test.py index 6aee1084..926cae7b 100644 --- a/code/test.py +++ b/code/test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import, print_function, division import sys import numpy @@ -137,12 +138,12 @@ def speed(): def time_test(m, l, idx, f, **kwargs): if not to_exec[idx]: return - print algo[idx] + print(algo[idx]) ts = m.call_time try: f(**kwargs) - except Exception, e: - print >> sys.stderr, 'test', algo[idx], 'FAILED', e + except Exception as e: + print('test', algo[idx], 'FAILED', e, file=sys.stderr) l.append(numpy.nan) return te = m.call_time @@ -265,7 +266,7 @@ def do_tests(): print >> sys.stderr, 'gpu % expected/get', ( expected_times_gpu / gpu_times) - print + print() if do_float64 and do_float32: print >> sys.stderr, 'float64/float32', ( float64_times / float32_times) @@ -286,7 +287,7 @@ def compare(x, y): # time and the real time, we consider this an error. return sum((ratio < 0.95) + (ratio > 1.05)) - print + print() if do_float64: err = compare(expected_times_64, float64_times) print >> sys.stderr, 'speed_failure_float64=' + str(err) From 4faede82e900555a063d6c7c385d0c3e59c04699 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Thu, 28 Jul 2016 10:29:44 -0400 Subject: [PATCH 269/417] python3 --- code/test.py | 130 +++++++++++++++++++++++++-------------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/code/test.py b/code/test.py index 926cae7b..4332e8b0 100644 --- a/code/test.py +++ b/code/test.py @@ -194,92 +194,92 @@ def do_tests(): theano.config.floatX = 'float64' theano.config.mode = 'FAST_RUN' float64_times = do_tests() - print >> sys.stderr, algo_executed - print >> sys.stderr, 'float64 times', float64_times - print >> sys.stderr, 'float64 expected', expected_times_64 - print >> sys.stderr, 'float64 % expected/get', ( - expected_times_64 / float64_times) + print(algo_executed, file=sys.stderr) + print('float64 times', float64_times, file=sys.stderr) + print('float64 expected', expected_times_64, file=sys.stderr) + print('float64 % expected/get', ( + expected_times_64 / float64_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the cpu theano.config.floatX = 'float32' if do_float32: float32_times = do_tests() - print >> sys.stderr, algo_executed - print >> sys.stderr, 'float32 times', float32_times - print >> sys.stderr, 'float32 expected', expected_times_32 - print >> sys.stderr, 'float32 % expected/get', ( - expected_times_32 / float32_times) + print(algo_executed, file=sys.stderr) + print('float32 times', float32_times, file=sys.stderr) + print('float32 expected', expected_times_32, file=sys.stderr) + print('float32 % expected/get', ( + expected_times_32 / float32_times), file=sys.stderr) if do_float64: - print >> sys.stderr, 'float64/float32', ( - float64_times / float32_times) - print >> sys.stderr - print >> sys.stderr, ('Duplicate the timing to have everything ' - 'in one place') - print >> sys.stderr, algo_executed - print >> sys.stderr, 'float64 times', float64_times - print >> sys.stderr, 'float64 expected', expected_times_64 - print >> sys.stderr, 'float64 % expected/get', ( - expected_times_64 / float64_times) - print >> sys.stderr, 'float32 times', float32_times - print >> sys.stderr, 'float32 expected', expected_times_32 - print >> sys.stderr, 'float32 % expected/get', ( - expected_times_32 / float32_times) - - print >> sys.stderr, 'float64/float32', ( - float64_times / float32_times) - print >> sys.stderr, 'expected float64/float32', ( - expected_times_64 / float32_times) + print('float64/float32', ( + float64_times / float32_times), file=sys.stderr) + print(file=sys.stderr) + print(('Duplicate the timing to have everything ' + 'in one place'), file=sys.stderr) + print(algo_executed, file=sys.stderr) + print('float64 times', float64_times, file=sys.stderr) + print('float64 expected', expected_times_64, file=sys.stderr) + print('float64 % expected/get', ( + expected_times_64 / float64_times), file=sys.stderr) + print('float32 times', float32_times, file=sys.stderr) + print('float32 expected', expected_times_32, file=sys.stderr) + print('float32 % expected/get', ( + expected_times_32 / float32_times), file=sys.stderr) + + print('float64/float32', ( + float64_times / float32_times), file=sys.stderr) + print('expected float64/float32', ( + expected_times_64 / float32_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the gpu import theano.sandbox.cuda if do_gpu: theano.sandbox.cuda.use('gpu') gpu_times = do_tests() - print >> sys.stderr, algo_executed - print >> sys.stderr, 'gpu times', gpu_times - print >> sys.stderr, 'gpu expected', expected_times_gpu - print >> sys.stderr, 'gpu % expected/get', ( - expected_times_gpu / gpu_times) + print(algo_executed, file=sys.stderr) + print('gpu times', gpu_times, file=sys.stderr) + print('gpu expected', expected_times_gpu, file=sys.stderr) + print('gpu % expected/get', ( + expected_times_gpu / gpu_times), file=sys.stderr) if do_float64: - print >> sys.stderr, 'float64/gpu', float64_times / gpu_times + print('float64/gpu', float64_times / gpu_times, file=sys.stderr) if (do_float64 + do_float32 + do_gpu) > 1: - print >> sys.stderr - print >> sys.stderr, ('Duplicate the timing to have everything ' - 'in one place') - print >> sys.stderr, algo_executed + print(file=sys.stderr) + print(('Duplicate the timing to have everything ' + 'in one place'), file=sys.stderr) + print(algo_executed, file=sys.stderr) if do_float64: - print >> sys.stderr, 'float64 times', float64_times - print >> sys.stderr, 'float64 expected', expected_times_64 - print >> sys.stderr, 'float64 % expected/get', ( - expected_times_64 / float64_times) + print('float64 times', float64_times, file=sys.stderr) + print('float64 expected', expected_times_64, file=sys.stderr) + print('float64 % expected/get', ( + expected_times_64 / float64_times), file=sys.stderr) if do_float32: - print >> sys.stderr, 'float32 times', float32_times - print >> sys.stderr, 'float32 expected', expected_times_32 - print >> sys.stderr, 'float32 % expected/get', ( - expected_times_32 / float32_times) + print('float32 times', float32_times, file=sys.stderr) + print('float32 expected', expected_times_32, file=sys.stderr) + print('float32 % expected/get', ( + expected_times_32 / float32_times), file=sys.stderr) if do_gpu: - print >> sys.stderr, 'gpu times', gpu_times - print >> sys.stderr, 'gpu expected', expected_times_gpu - print >> sys.stderr, 'gpu % expected/get', ( - expected_times_gpu / gpu_times) + print('gpu times', gpu_times, file=sys.stderr) + print('gpu expected', expected_times_gpu, file=sys.stderr) + print('gpu % expected/get', ( + expected_times_gpu / gpu_times), file=sys.stderr) print() if do_float64 and do_float32: - print >> sys.stderr, 'float64/float32', ( - float64_times / float32_times) - print >> sys.stderr, 'expected float64/float32', ( - expected_times_64 / float32_times) + print('float64/float32', ( + float64_times / float32_times), file=sys.stderr) + print('expected float64/float32', ( + expected_times_64 / float32_times), file=sys.stderr) if do_float64 and do_gpu: - print >> sys.stderr, 'float64/gpu', float64_times / gpu_times - print >> sys.stderr, 'expected float64/gpu', ( - expected_times_64 / gpu_times) + print('float64/gpu', float64_times / gpu_times, file=sys.stderr) + print('expected float64/gpu', ( + expected_times_64 / gpu_times), file=sys.stderr) if do_float32 and do_gpu: - print >> sys.stderr, 'float32/gpu', float32_times / gpu_times - print >> sys.stderr, 'expected float32/gpu', ( - expected_times_32 / gpu_times) + print('float32/gpu', float32_times / gpu_times, file=sys.stderr) + print('expected float32/gpu', ( + expected_times_32 / gpu_times), file=sys.stderr) def compare(x, y): ratio = x / y @@ -287,15 +287,15 @@ def compare(x, y): # time and the real time, we consider this an error. return sum((ratio < 0.95) + (ratio > 1.05)) - print() + print(file=sys.stderr) if do_float64: err = compare(expected_times_64, float64_times) - print >> sys.stderr, 'speed_failure_float64=' + str(err) + print('speed_failure_float64=' + str(err), file=sys.stderr) if do_float32: err = compare(expected_times_32, float32_times) - print >> sys.stderr, 'speed_failure_float32=' + str(err) + print('speed_failure_float32=' + str(err), file=sys.stderr) if do_gpu: err = compare(expected_times_gpu, gpu_times) - print >> sys.stderr, 'speed_failure_gpu=' + str(err) + print('speed_failure_gpu=' + str(err), file=sys.stderr) assert not numpy.isnan(gpu_times).any() From ac029111f94c67c480746ebd23229af099fd2570 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Wed, 31 Aug 2016 11:18:18 -0400 Subject: [PATCH 270/417] unzip -f to avoid prompt in data download --- data/download.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/download.sh b/data/download.sh index 160b0986..67c5c057 100755 --- a/data/download.sh +++ b/data/download.sh @@ -15,8 +15,8 @@ fi $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist_py3k.pkl.gz -$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip imdb.pkl.gz -$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.dict.pkl.gz && gunzip imdb.dict.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.pkl.gz && gunzip -f imdb.pkl.gz +$DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/imdb.dict.pkl.gz && gunzip -f imdb.dict.pkl.gz $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/data/Nottingham.zip && unzip -u Nottingham.zip $DL_CMD http://www.iro.umontreal.ca/~lisa/deep/midi.zip && unzip -u midi.zip -d ../code && echo "extracted Modified Python MIDI package (GPL)" $DL_CMD http://lisaweb.iro.umontreal.ca/transfert/lisa/users/mesnilgr/atis/atis.fold0.pkl.gz From f6db4f12f191a421f7a0f948d68cce36290fb617 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Wed, 7 Sep 2016 10:25:51 -0400 Subject: [PATCH 271/417] change compiledir and add xunit for jenkins --- misc/do_nightly_build | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/misc/do_nightly_build b/misc/do_nightly_build index bd703f04..cafab51c 100755 --- a/misc/do_nightly_build +++ b/misc/do_nightly_build @@ -1,9 +1,15 @@ #!/bin/bash -#we set the compiledir to the /Tmp dir to make the test faster by bypassing the nfs network. + +# If not jenkins, set workspace to local Tmp +if [ -v $WORKSPACE ]; then + WORKSPACE=/Tmp +fi + date -ROOT_CWD=/Tmp/nightly_build -COMPILEDIR=/Tmp/lisa_theano_compile_dir_deeplearning +ROOT_CWD=$WORKSPACE/nightly_build +COMPILEDIR=$WORKSPACE/lisa_theano_compile_dir_deeplearning NOSETESTS=${ROOT_CWD}/Theano/bin/theano-nose +XUNIT="--with-xunit --xunit-file=" FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} export PYTHONPATH=${ROOT_CWD}/Theano:${ROOT_CWD}/Pylearn:$PYTHONPATH @@ -19,14 +25,17 @@ echo "git version:" `git rev-parse HEAD` #echo "executing nosetests with mode=FAST_COMPILE" #THEANO_FLAGS=${FLAGS},mode=FAST_COMPILE ${NOSETESTS} echo "executing nosetests speed with mode=FAST_RUN" -THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} test.py:speed +FILE=${ROOT_CWD}/dlt_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} test.py:speed #echo "executing nosetests speed with mode=FAST_RUN and OMP_NUM_THREADS=2" #OMP_NUM_THREADS=2 THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} test.py:speed echo "executing nosetests with mode=FAST_RUN,floatX=float32" -THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} +FILE=${ROOT_CWD}/dlt_32bit_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} #we change the seed and record it everyday to test different combination. We record it to be able to reproduce bug caused by different seed. We don't want multiple test in DEBUG_MODE each day as this take too long. #seed=$RANDOM #echo "executing nosetests with mode=DEBUG_MODE with seed of the day $seed" -#THEANO_DEBUGMODE_CHECK_STRIDES=0 THEANO_DEBUGMODE_PATIENCE=3 THEANO_COMPILEDIR=/Tmp/lisa_theano_compile_dir_deeplearning THEANO_UNITTEST_SEED=$seed THEANO_DEFAULT_MODE=DEBUG_MODE ${NOSETESTS} +#FILE=${ROOT_CWD}/'dlt_debug_tests.xml' +#THEANO_DEBUGMODE_CHECK_STRIDES=0 THEANO_DEBUGMODE_PATIENCE=3 THEANO_COMPILEDIR=$WORKSPACE/lisa_theano_compile_dir_deeplearning THEANO_UNITTEST_SEED=$seed THEANO_DEFAULT_MODE=DEBUG_MODE ${NOSETESTS} ${XUNIT}${FILE} From a0362806a029f20d7ed920868ded79d1b388d741 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Wed, 7 Sep 2016 19:43:00 -0400 Subject: [PATCH 272/417] dtl compiledir --- misc/do_nightly_build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/do_nightly_build b/misc/do_nightly_build index cafab51c..a8ee32cf 100755 --- a/misc/do_nightly_build +++ b/misc/do_nightly_build @@ -7,7 +7,7 @@ fi date ROOT_CWD=$WORKSPACE/nightly_build -COMPILEDIR=$WORKSPACE/lisa_theano_compile_dir_deeplearning +COMPILEDIR=$WORKSPACE/compile/lisa_theano_compile_dir_deeplearning NOSETESTS=${ROOT_CWD}/Theano/bin/theano-nose XUNIT="--with-xunit --xunit-file=" From 31e194d4a844db9455cbb72a91b0e717084f84ed Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 9 Sep 2016 15:45:53 -0400 Subject: [PATCH 273/417] use TMPDIR for buildbot --- misc/do_nightly_build | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/misc/do_nightly_build b/misc/do_nightly_build index a8ee32cf..29281050 100755 --- a/misc/do_nightly_build +++ b/misc/do_nightly_build @@ -2,7 +2,10 @@ # If not jenkins, set workspace to local Tmp if [ -v $WORKSPACE ]; then - WORKSPACE=/Tmp + if [ -v $TMPDIR ]; then + TMPDIR=/tmp + fi + WORKSPACE=$TMPDIR fi date From 80b969171df5bb341788864a46e433aa06858ccb Mon Sep 17 00:00:00 2001 From: slefrancois Date: Mon, 12 Sep 2016 09:36:55 -0400 Subject: [PATCH 274/417] test file name to float32 --- misc/do_nightly_build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/do_nightly_build b/misc/do_nightly_build index 29281050..ef2b8319 100755 --- a/misc/do_nightly_build +++ b/misc/do_nightly_build @@ -33,7 +33,7 @@ THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} test.py:speed #echo "executing nosetests speed with mode=FAST_RUN and OMP_NUM_THREADS=2" #OMP_NUM_THREADS=2 THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} test.py:speed echo "executing nosetests with mode=FAST_RUN,floatX=float32" -FILE=${ROOT_CWD}/dlt_32bit_tests.xml +FILE=${ROOT_CWD}/dlt_float32_tests.xml THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} #we change the seed and record it everyday to test different combination. We record it to be able to reproduce bug caused by different seed. We don't want multiple test in DEBUG_MODE each day as this take too long. From 793d6181bc70b45a5d7521131822c62d78d9a418 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Mon, 19 Sep 2016 12:06:05 -0400 Subject: [PATCH 275/417] add jenkins buildbot script --- .jenkins/jenkins_buildbot_dlt.sh | 35 ++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 .jenkins/jenkins_buildbot_dlt.sh diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh new file mode 100755 index 00000000..0d2e49f2 --- /dev/null +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +BUILDBOT_DIR=$WORKSPACE/nightly_build +source $HOME/.bashrc + +mkdir -p ${BUILDBOT_DIR} + +date +COMPILEDIR=$WORKSPACE/compile/lisa_theano_compile_dir_deeplearning +NOSETESTS=${BUILDBOT_DIR}/Theano/bin/theano-nose +XUNIT="--with-xunit --xunit-file=" + +FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} +export PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/Pylearn:$PYTHONPATH + +cd ${BUILDBOT_DIR} +if [ ! -d ${BUILDBOT_DIR}/Theano ]; then + git clone git://github.com/Theano/Theano.git +fi +# update repo +cd ${BUILDBOT_DIR}/Theano; git pull + +${WORKSPACE}/data/download.sh + +cd ${BUILDBOT_DIR}/Theano +echo "git version for Theano:" `git rev-parse HEAD` +cd ${WORKSPACE}/code +echo "git version:" `git rev-parse HEAD` + +echo "executing nosetests speed with mode=FAST_RUN" +FILE=${BUILDBOT_DIR}/dlt_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} test.py:speed +echo "executing nosetests with mode=FAST_RUN,floatX=float32" +FILE=${BUILDBOT_DIR}/dlt_float32_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} From 12fb33662170918912f473e75360434b4680c7b9 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Mon, 19 Sep 2016 14:33:42 -0400 Subject: [PATCH 276/417] midi --- .jenkins/jenkins_buildbot_dlt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 0d2e49f2..2cd41d5e 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -20,7 +20,8 @@ fi # update repo cd ${BUILDBOT_DIR}/Theano; git pull -${WORKSPACE}/data/download.sh +cd ${WORKSPACE}/data +./download.sh cd ${BUILDBOT_DIR}/Theano echo "git version for Theano:" `git rev-parse HEAD` From 93c9a3642d8952f7816273cddfc55a5a9f64077b Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 3 Oct 2016 12:24:53 -0400 Subject: [PATCH 277/417] Use MRG_RandomStreams instead for shared_randomstreams for GPU compat. --- code/SdA.py | 2 +- code/dA.py | 2 +- code/hmc/hmc.py | 2 +- code/rbm.py | 2 +- code/rnnrbm.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index 3d9589ac..eb7b7357 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -40,7 +40,7 @@ import theano import theano.tensor as T -from theano.tensor.shared_randomstreams import RandomStreams +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams from logistic_sgd import LogisticRegression, load_data from mlp import HiddenLayer diff --git a/code/dA.py b/code/dA.py index 0d9efa54..aad3d454 100644 --- a/code/dA.py +++ b/code/dA.py @@ -40,7 +40,7 @@ import theano import theano.tensor as T -from theano.tensor.shared_randomstreams import RandomStreams +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams from logistic_sgd import load_data from utils import tile_raster_images diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index aeb49937..f16a50c1 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -358,7 +358,7 @@ def new_from_shared_positions( stepsize = sharedX(initial_stepsize, 'hmc_stepsize') avg_acceptance_rate = sharedX(target_acceptance_rate, 'avg_acceptance_rate') - s_rng = TT.shared_randomstreams.RandomStreams(seed) + s_rng = theano.sandbox.rng_mrg.MRG_RandomStreams(seed) # define graph for an `n_steps` HMC simulation accept, final_pos = hmc_move( diff --git a/code/rbm.py b/code/rbm.py index 3800cca7..6e4f1012 100644 --- a/code/rbm.py +++ b/code/rbm.py @@ -20,7 +20,7 @@ import theano.tensor as T import os -from theano.tensor.shared_randomstreams import RandomStreams +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams from utils import tile_raster_images from logistic_sgd import load_data diff --git a/code/rnnrbm.py b/code/rnnrbm.py index b8420b9b..900ffdc6 100644 --- a/code/rnnrbm.py +++ b/code/rnnrbm.py @@ -19,7 +19,7 @@ from midi.utils import midiread, midiwrite import theano import theano.tensor as T -from theano.tensor.shared_randomstreams import RandomStreams +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams #Don't use a python long as this don't work on 32 bits computers. numpy.random.seed(0xbeef) From 4f251cd72dac2754c173c0a850f215b73fdb19f5 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Thu, 6 Oct 2016 12:01:01 -0400 Subject: [PATCH 278/417] add testsuites names --- .jenkins/jenkins_buildbot_dlt.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 2cd41d5e..a4e4b1e4 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -9,6 +9,8 @@ date COMPILEDIR=$WORKSPACE/compile/lisa_theano_compile_dir_deeplearning NOSETESTS=${BUILDBOT_DIR}/Theano/bin/theano-nose XUNIT="--with-xunit --xunit-file=" +# name test suites +SUITE="--xunit-prefix-with-testsuite-name --xunit-testsuite-name=" FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} export PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/Pylearn:$PYTHONPATH @@ -29,8 +31,10 @@ cd ${WORKSPACE}/code echo "git version:" `git rev-parse HEAD` echo "executing nosetests speed with mode=FAST_RUN" -FILE=${BUILDBOT_DIR}/dlt_tests.xml -THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} test.py:speed +NAME=dlt_speed +FILE=${BUILDBOT_DIR}/${NAME}_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} ${SUITE}${NAME} test.py:speed echo "executing nosetests with mode=FAST_RUN,floatX=float32" -FILE=${BUILDBOT_DIR}/dlt_float32_tests.xml -THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} +NAME=dlt_float32 +FILE=${BUILDBOT_DIR}/${NAME}_tests.xml +THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} ${SUITE}${NAME} From 82c85e2a6f74a92736c2afdd805710dedfcc4f4f Mon Sep 17 00:00:00 2001 From: slefrancois Date: Thu, 6 Oct 2016 14:36:46 -0400 Subject: [PATCH 279/417] Add JUnit writer for speed tests, remove hardcoded reference times --- code/test.py | 117 +++++++++++++-------------------------------------- 1 file changed, 29 insertions(+), 88 deletions(-) diff --git a/code/test.py b/code/test.py index 4332e8b0..60c0af02 100644 --- a/code/test.py +++ b/code/test.py @@ -98,43 +98,7 @@ def speed(): do_gpu = True algo_executed = [s for idx, s in enumerate(algo) if to_exec[idx]] - #Timming expected are from the buildbot that have an i7-920 @ - # 2.67GHz with hyperthread enabled for the cpu, 12G of ram. An GeForce GTX - # 580 for the GPU. OS=Fedora 14, gcc=4.5.1, python/BLAS from EPD - # 7.1-2 (python 2.7.2, mkl unknow). BLAS with only 1 thread. - - expected_times_64 = numpy.asarray([9.3, 21.0, 76.1, 73.7, 116.4, - 346.9, 355.0, 268.2, 115.8, 16.8, 91.6]) - expected_times_32 = numpy.asarray([6.4, 14.7, 42.5, 63.1, 71, - 191.2, 199.0, 201.9, 107, 12.6, 61.3]) - - # Number with just 1 decimal are new value that are faster with - # the Theano version 0.5rc2 Other number are older. They are not - # updated, as we where faster in the past! - # TODO: find why and fix this! - -# Here is the value for the buildbot on February 3th 2012 with a GTX 285 -# sgd, cg mlp conv da -# sda dbn rbm -# gpu times[3.72957802, 9.94316864, 29.1772666, 9.13857198, 25.91144657, -# 18.30802011, 53.38651466, 285.41386175] -# expected [3.076634879, 7.555234910, 18.99226785, 9.58915591, 24.130070450, -# 24.77524018, 92.66246653, 322.340329170] -# sgd, cg mlp conv da -# sda dbn rbm -#expected/get [0.82492841, 0.75984178, 0.65092691, 1.04930573, 0.93125138 -# 1.35324519 1.7356905 1.12937868] - - expected_times_gpu = numpy.asarray([2.9, 7.55523491, 18.99226785, - 5.8, 19.2, - 11.2, 7.3, 122, 112.5, 31.1, 8.3]) - expected_times_64 = [s for idx, s in enumerate(expected_times_64) - if to_exec[idx]] - expected_times_32 = [s for idx, s in enumerate(expected_times_32) - if to_exec[idx]] - expected_times_gpu = [s for idx, s in enumerate(expected_times_gpu) - if to_exec[idx]] - + def time_test(m, l, idx, f, **kwargs): if not to_exec[idx]: return @@ -196,9 +160,6 @@ def do_tests(): float64_times = do_tests() print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the cpu theano.config.floatX = 'float32' @@ -206,9 +167,6 @@ def do_tests(): float32_times = do_tests() print(algo_executed, file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) if do_float64: print('float64/float32', ( @@ -218,18 +176,10 @@ def do_tests(): 'in one place'), file=sys.stderr) print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) print('float64/float32', ( float64_times / float32_times), file=sys.stderr) - print('expected float64/float32', ( - expected_times_64 / float32_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the gpu import theano.sandbox.cuda @@ -238,9 +188,6 @@ def do_tests(): gpu_times = do_tests() print(algo_executed, file=sys.stderr) print('gpu times', gpu_times, file=sys.stderr) - print('gpu expected', expected_times_gpu, file=sys.stderr) - print('gpu % expected/get', ( - expected_times_gpu / gpu_times), file=sys.stderr) if do_float64: print('float64/gpu', float64_times / gpu_times, file=sys.stderr) @@ -252,50 +199,44 @@ def do_tests(): print(algo_executed, file=sys.stderr) if do_float64: print('float64 times', float64_times, file=sys.stderr) - print('float64 expected', expected_times_64, file=sys.stderr) - print('float64 % expected/get', ( - expected_times_64 / float64_times), file=sys.stderr) if do_float32: print('float32 times', float32_times, file=sys.stderr) - print('float32 expected', expected_times_32, file=sys.stderr) - print('float32 % expected/get', ( - expected_times_32 / float32_times), file=sys.stderr) if do_gpu: print('gpu times', gpu_times, file=sys.stderr) - print('gpu expected', expected_times_gpu, file=sys.stderr) - print('gpu % expected/get', ( - expected_times_gpu / gpu_times), file=sys.stderr) print() if do_float64 and do_float32: print('float64/float32', ( float64_times / float32_times), file=sys.stderr) - print('expected float64/float32', ( - expected_times_64 / float32_times), file=sys.stderr) if do_float64 and do_gpu: print('float64/gpu', float64_times / gpu_times, file=sys.stderr) - print('expected float64/gpu', ( - expected_times_64 / gpu_times), file=sys.stderr) if do_float32 and do_gpu: print('float32/gpu', float32_times / gpu_times, file=sys.stderr) - print('expected float32/gpu', ( - expected_times_32 / gpu_times), file=sys.stderr) - - def compare(x, y): - ratio = x / y - # If there is more then 5% difference between the expected - # time and the real time, we consider this an error. - return sum((ratio < 0.95) + (ratio > 1.05)) - - print(file=sys.stderr) - if do_float64: - err = compare(expected_times_64, float64_times) - print('speed_failure_float64=' + str(err), file=sys.stderr) - if do_float32: - err = compare(expected_times_32, float32_times) - print('speed_failure_float32=' + str(err), file=sys.stderr) - if do_gpu: - err = compare(expected_times_gpu, gpu_times) - print('speed_failure_gpu=' + str(err), file=sys.stderr) - - assert not numpy.isnan(gpu_times).any() + + # Write JUnit xml for speed test performance report + + speed_file = 'speedtests_time.xml' + + # Define speed test file write method + def write_junit(filename, algos, times, label): + with open(filename, 'a') as f: + for algo, time in zip(algos, times): + f.write(' ' + .format(label=label, algo=algo, time=time)) + f.write(' \n') + + test_total = numpy.size(float64_times) \ + + numpy.size(float32_times) \ + + numpy.size(gpu_times) + + with open(speed_file, 'w') as f: + f.write('\n') + f.write('\n' + .format(ntests=numpy.size(test_total))) + + write_junit(speed_file, algo_executed, float64_times, label='float64') + write_junit(speed_file, algo_executed, float32_times, label='float32') + write_junit(speed_file, algo_executed, gpu_times, label='gpu') + + with open(speed_file, 'a') as f: + f.write('\n') From 9918b7a9d377af71ac1323187913861651b26ce8 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 7 Oct 2016 09:04:14 -0400 Subject: [PATCH 280/417] remove testsuite prefix option --- .jenkins/jenkins_buildbot_dlt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index a4e4b1e4..846cf7fc 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -10,7 +10,7 @@ COMPILEDIR=$WORKSPACE/compile/lisa_theano_compile_dir_deeplearning NOSETESTS=${BUILDBOT_DIR}/Theano/bin/theano-nose XUNIT="--with-xunit --xunit-file=" # name test suites -SUITE="--xunit-prefix-with-testsuite-name --xunit-testsuite-name=" +SUITE="--xunit-testsuite-name=" FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} export PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/Pylearn:$PYTHONPATH From f14107d31d5cb05f192129a95d3f272acf4dbc09 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 7 Oct 2016 17:28:37 -0400 Subject: [PATCH 281/417] single performance file open, correct only access times variables if tests ran --- code/test.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/code/test.py b/code/test.py index 60c0af02..22a59655 100644 --- a/code/test.py +++ b/code/test.py @@ -152,12 +152,18 @@ def do_tests(): saveto='') return numpy.asarray(l) + # Initialize test count and results dictionnary + test_total = 0 + times_dic = {} + #test in float64 in FAST_RUN mode on the cpu import theano if do_float64: theano.config.floatX = 'float64' theano.config.mode = 'FAST_RUN' float64_times = do_tests() + times_dic['float64'] = float64_times + test_total += numpy.size(float64_times) print(algo_executed, file=sys.stderr) print('float64 times', float64_times, file=sys.stderr) @@ -165,6 +171,8 @@ def do_tests(): theano.config.floatX = 'float32' if do_float32: float32_times = do_tests() + times_dic['float32'] = float32_times + test_total += numpy.size(float32_times) print(algo_executed, file=sys.stderr) print('float32 times', float32_times, file=sys.stderr) @@ -186,6 +194,8 @@ def do_tests(): if do_gpu: theano.sandbox.cuda.use('gpu') gpu_times = do_tests() + times_dic['gpu'] = gpu_times + test_total += numpy.size(gpu_times) print(algo_executed, file=sys.stderr) print('gpu times', gpu_times, file=sys.stderr) @@ -213,30 +223,18 @@ def do_tests(): if do_float32 and do_gpu: print('float32/gpu', float32_times / gpu_times, file=sys.stderr) - # Write JUnit xml for speed test performance report - - speed_file = 'speedtests_time.xml' - - # Define speed test file write method - def write_junit(filename, algos, times, label): - with open(filename, 'a') as f: - for algo, time in zip(algos, times): - f.write(' ' - .format(label=label, algo=algo, time=time)) - f.write(' \n') - - test_total = numpy.size(float64_times) \ - + numpy.size(float32_times) \ - + numpy.size(gpu_times) - - with open(speed_file, 'w') as f: + # Generate JUnit performance report + # Define speedtest file write method + def write_junit(f, algos, times, label): + for algo, time in zip(algos, times): + f.write(' ' + .format(label=label, algo=algo, time=time)) + f.write(' \n') + + with open('speedtests_time.xml', 'w') as f: f.write('\n') - f.write('\n' - .format(ntests=numpy.size(test_total))) - - write_junit(speed_file, algo_executed, float64_times, label='float64') - write_junit(speed_file, algo_executed, float32_times, label='float32') - write_junit(speed_file, algo_executed, gpu_times, label='gpu') - - with open(speed_file, 'a') as f: + f.write('\n' + .format(ntests=test_total)) + for label, times in times_dic.items(): + write_junit(f, algo_executed, times, label) f.write('\n') From f724c2c6054c736c548196d2a7a000ec307e0b0d Mon Sep 17 00:00:00 2001 From: slefrancois Date: Fri, 7 Oct 2016 17:36:44 -0400 Subject: [PATCH 282/417] move assert gpu_times not nan --- code/test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/test.py b/code/test.py index 22a59655..b3077b7c 100644 --- a/code/test.py +++ b/code/test.py @@ -238,3 +238,6 @@ def write_junit(f, algos, times, label): for label, times in times_dic.items(): write_junit(f, algo_executed, times, label) f.write('\n') + + if do_gpu: + assert not numpy.isnan(gpu_times).any() From 85f56c22ebf46e260e38215d32e7f893e95fdcc3 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Thu, 13 Oct 2016 13:14:11 -0400 Subject: [PATCH 283/417] add explicit CUDA path to buildbot --- .jenkins/jenkins_buildbot_dlt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 846cf7fc..283eb933 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -1,7 +1,11 @@ #!/bin/bash +# CUDA +export PATH=/usr/local/cuda/bin:$PATH +export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH +export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH + BUILDBOT_DIR=$WORKSPACE/nightly_build -source $HOME/.bashrc mkdir -p ${BUILDBOT_DIR} From d4035919fe2342ba83f104e34d13a8962203c1e6 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 20 Oct 2016 14:22:16 -0400 Subject: [PATCH 284/417] Compute mean in higher precision to avoid overflow. --- code/DBN.py | 6 +++--- code/dA.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/code/DBN.py b/code/DBN.py index 3b2bd230..e1bb66df 100644 --- a/code/DBN.py +++ b/code/DBN.py @@ -340,7 +340,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr)) print('Pre-training layer %i, epoch %d, cost ' % (i, epoch), end=' ') - print(numpy.mean(c)) + print(numpy.mean(c, dtype='float64')) end_time = timeit.default_timer() # end-snippet-2 @@ -391,7 +391,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() - this_validation_loss = numpy.mean(validation_losses) + this_validation_loss = numpy.mean(validation_losses, dtype='float64') print('epoch %i, minibatch %i/%i, validation error %f %%' % ( epoch, minibatch_index + 1, @@ -414,7 +414,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100, # test it on the test set test_losses = test_model() - test_score = numpy.mean(test_losses) + test_score = numpy.mean(test_losses, dtype='float64') print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, diff --git a/code/dA.py b/code/dA.py index aad3d454..93a696f8 100644 --- a/code/dA.py +++ b/code/dA.py @@ -336,7 +336,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, for batch_index in range(n_train_batches): c.append(train_da(batch_index)) - print('Training epoch %d, cost ' % epoch, numpy.mean(c)) + print('Training epoch %d, cost ' % epoch, numpy.mean(c, dtype='float64')) end_time = timeit.default_timer() @@ -394,7 +394,7 @@ def test_dA(learning_rate=0.1, training_epochs=15, for batch_index in range(n_train_batches): c.append(train_da(batch_index)) - print('Training epoch %d, cost ' % epoch, numpy.mean(c)) + print('Training epoch %d, cost ' % epoch, numpy.mean(c, dtype='float64')) end_time = timeit.default_timer() From 5a13d9869587a84018b939f83f5fd85293c9a8a1 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Thu, 20 Oct 2016 16:18:46 -0400 Subject: [PATCH 285/417] Fix import of sandbox. --- code/hmc/hmc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/code/hmc/hmc.py b/code/hmc/hmc.py index f16a50c1..cf4d20a1 100644 --- a/code/hmc/hmc.py +++ b/code/hmc/hmc.py @@ -7,6 +7,7 @@ from theano import function, shared from theano import tensor as TT import theano +import theano.sandbox.rng_mrg sharedX = (lambda X, name: shared(numpy.asarray(X, dtype=theano.config.floatX), name=name)) @@ -275,14 +276,14 @@ def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, """ - ## POSITION UPDATES ## + # POSITION UPDATES # # broadcast `accept` scalar to tensor with the same dimensions as # final_pos. accept_matrix = accept.dimshuffle(0, *(('x',) * (final_pos.ndim - 1))) # if accept is True, update to `final_pos` else stay put new_positions = TT.switch(accept_matrix, final_pos, positions) # end-snippet-5 start-snippet-7 - ## STEPSIZE UPDATES ## + # STEPSIZE UPDATES # # if acceptance rate is too low, our sampler is too "noisy" and we reduce # the stepsize. If it is too high, our sampler is too conservative, we can # get away with a larger stepsize (resulting in better mixing). @@ -292,7 +293,7 @@ def hmc_updates(positions, stepsize, avg_acceptance_rate, final_pos, accept, new_stepsize = TT.clip(_new_stepsize, stepsize_min, stepsize_max) # end-snippet-7 start-snippet-6 - ## ACCEPT RATE UPDATES ## + # ACCEPT RATE UPDATES # # perform exponential moving average mean_dtype = theano.scalar.upcast(accept.dtype, avg_acceptance_rate.dtype) new_acceptance_rate = TT.add( From 93837e03aeeff6917d2b3a121e05341b663fa890 Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Mon, 24 Oct 2016 16:33:18 -0400 Subject: [PATCH 286/417] Fix printout in lstm.py. --- code/lstm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/lstm.py b/code/lstm.py index 1c285928..a3010a9f 100644 --- a/code/lstm.py +++ b/code/lstm.py @@ -605,8 +605,8 @@ def train_lstm( best_p = unzip(tparams) bad_counter = 0 - print( ('Train ', train_err, 'Valid ', valid_err, - 'Test ', test_err) ) + print('Train ', train_err, 'Valid ', valid_err, + 'Test ', test_err) if (len(history_errs) > patience and valid_err >= numpy.array(history_errs)[:-patience, From 780cecc9abbe6181e8fe37f9bda386bdc01fe2ec Mon Sep 17 00:00:00 2001 From: Arnaud Bergeron Date: Wed, 26 Oct 2016 14:46:32 -0400 Subject: [PATCH 287/417] Adjust mean dtypes for scores in SdA too. --- code/SdA.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/code/SdA.py b/code/SdA.py index eb7b7357..8da74797 100644 --- a/code/SdA.py +++ b/code/SdA.py @@ -394,7 +394,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, c.append(pretraining_fns[i](index=batch_index, corruption=corruption_levels[i], lr=pretrain_lr)) - print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, numpy.mean(c))) + print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, numpy.mean(c, dtype='float64'))) end_time = timeit.default_timer() @@ -442,7 +442,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, if (iter + 1) % validation_frequency == 0: validation_losses = validate_model() - this_validation_loss = numpy.mean(validation_losses) + this_validation_loss = numpy.mean(validation_losses, dtype='float64') print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) @@ -463,7 +463,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15, # test it on the test set test_losses = test_model() - test_score = numpy.mean(test_losses) + test_score = numpy.mean(test_losses, dtype='float64') print((' epoch %i, minibatch %i/%i, test error of ' 'best model %f %%') % (epoch, minibatch_index + 1, n_train_batches, From cd462eccb4f351cec6915c4294b0197fd2aa51d9 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Thu, 3 Nov 2016 15:20:16 -0400 Subject: [PATCH 288/417] split performance report file --- code/test.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/code/test.py b/code/test.py index b3077b7c..c2ad68bc 100644 --- a/code/test.py +++ b/code/test.py @@ -224,20 +224,16 @@ def do_tests(): print('float32/gpu', float32_times / gpu_times, file=sys.stderr) # Generate JUnit performance report - # Define speedtest file write method - def write_junit(f, algos, times, label): - for algo, time in zip(algos, times): - f.write(' ' - .format(label=label, algo=algo, time=time)) - f.write(' \n') - - with open('speedtests_time.xml', 'w') as f: - f.write('\n') - f.write('\n' - .format(ntests=test_total)) - for label, times in times_dic.items(): - write_junit(f, algo_executed, times, label) - f.write('\n') + for label, times in times_dic.items(): + with open('speedtests_{label}.xml'.format(label=label), 'w') as f: + f.write('\n') + f.write('\n' + .format(label=label, ntests=test_total/len(times_dic))) + for algo, time in zip(algo_executed, times): + f.write(' ' + .format(label=label, algo=algo, time=time)) + f.write(' \n') + f.write('\n') if do_gpu: assert not numpy.isnan(gpu_times).any() From fd5cb65460df2dee9cfa250e3e7fbc864720bd86 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Fri, 13 Jan 2017 15:20:23 -0500 Subject: [PATCH 289/417] Do the speed test on the new gpu back-end. --- code/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/test.py b/code/test.py index c2ad68bc..5053b8c4 100644 --- a/code/test.py +++ b/code/test.py @@ -190,9 +190,9 @@ def do_tests(): float64_times / float32_times), file=sys.stderr) #test in float32 in FAST_RUN mode on the gpu - import theano.sandbox.cuda + import theano.gpuarray if do_gpu: - theano.sandbox.cuda.use('gpu') + theano.gpuarray.use('cuda') gpu_times = do_tests() times_dic['gpu'] = gpu_times test_total += numpy.size(gpu_times) From e481d33b2492e37274c2db8389f3b5452767dd68 Mon Sep 17 00:00:00 2001 From: slefrancois Date: Mon, 16 Jan 2017 10:05:22 -0500 Subject: [PATCH 290/417] install libgpuarray for dlt speed tests --- .jenkins/jenkins_buildbot_dlt.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 283eb933..243cd4ef 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -19,6 +19,38 @@ SUITE="--xunit-testsuite-name=" FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR} export PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/Pylearn:$PYTHONPATH +# Install libgpuarray and pygpu +cd ${BUILDBOT_DIR} + +# Make fresh clone (with no history since we don't need it) +rm -rf libgpuarray +git clone --depth 1 "https://github.com/Theano/libgpuarray.git" + +(cd libgpuarray && echo "libgpuarray commit" && git rev-parse HEAD) + +# Clean up previous installs (to make sure no old files are left) +rm -rf local +mkdir local + +# Build libgpuarray and run C tests +mkdir libgpuarray/build +(cd libgpuarray/build && cmake .. -DCMAKE_BUILD_TYPE=${GPUARRAY_CONFIG} -DCMAKE_INSTALL_PREFIX=${BUILDBOT_DIR}/local && make) + +# Finally install +(cd libgpuarray/build && make install) +export LD_LIBRARY_PATH=${BUILDBOT_DIR}/local/lib:${LD_LIBRARY_PATH} +export LIBRARY_PATH=${BUILDBOT_DIR}/local/lib:${LIBRARY_PATH} +export CPATH=${BUILDBOT_DIR}/local/include:${CPATH} + +# Build the pygpu modules +(cd libgpuarray && python setup.py build_ext --inplace -I${BUILDBOT_DIR}/local/include -L${BUILDBOT_DIR}/local/lib) + +mkdir ${BUILDBOT_DIR}/local/lib/python +export PYTHONPATH=${PYTHONPATH}:${BUILDBOT_DIR}/local/lib/python +# Then install +(cd libgpuarray && python setup.py install --home=${BUILDBOT_DIR}/local) + +# Install Theano cd ${BUILDBOT_DIR} if [ ! -d ${BUILDBOT_DIR}/Theano ]; then git clone git://github.com/Theano/Theano.git From 73e621d37ae6bb7f0747e831822f39435e61bab1 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Tue, 18 Apr 2017 09:59:43 -0400 Subject: [PATCH 291/417] move speedtest cache outside workspace --- .DS_Store | Bin 0 -> 6148 bytes .jenkins/jenkins_buildbot_dlt.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..3cd979e05c0d9c2d21079f88c5fedc75d991437e GIT binary patch literal 6148 zcmeHKF=_)r43v^93~5}Z+%Mz@i*a7y57@*ZO|Zcvsjter{4~! Date: Wed, 19 Apr 2017 14:52:56 -0400 Subject: [PATCH 292/417] add label to speedtest class --- code/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/test.py b/code/test.py index 5053b8c4..8768d8c1 100644 --- a/code/test.py +++ b/code/test.py @@ -230,7 +230,7 @@ def do_tests(): f.write('\n' .format(label=label, ntests=test_total/len(times_dic))) for algo, time in zip(algo_executed, times): - f.write(' ' + f.write(' ' .format(label=label, algo=algo, time=time)) f.write(' \n') f.write('\n') From e7b2dc866d8a460cf5e0f20805fa4155649f1840 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Fri, 28 Apr 2017 14:10:43 -0400 Subject: [PATCH 293/417] buildbot includes theano.gpuarray --- .DS_Store | Bin 6148 -> 0 bytes .jenkins/jenkins_buildbot_dlt.sh | 10 ++++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 3cd979e05c0d9c2d21079f88c5fedc75d991437e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKF=_)r43v^93~5}Z+%Mz@i*a7y57@*ZO|Zcvsjter{4~! Date: Tue, 6 Jun 2017 14:56:35 -0400 Subject: [PATCH 294/417] fix typos/spelling --- doc/gettingstarted.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index d765f14a..85111d11 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -147,7 +147,7 @@ MNIST Dataset The data has to be stored as floats on the GPU ( the right ``dtype`` for storing on the GPU is given by ``theano.config.floatX``). -To get around this shortcomming for the labels, we store them as float, +To get around this shortcoming for the labels, we store them as float, and then cast it to int. .. note:: @@ -316,7 +316,7 @@ The likelihood of the correct class is not the same as the number of right predictions, but from the point of view of a randomly initialized classifier they are pretty similar. Remember that likelihood and zero-one loss are different objectives; -you should see that they are corralated on the validation set but +you should see that they are correlated on the validation set but sometimes one will rise while the other falls, or vice-versa. Since we usually speak in terms of minimizing a loss function, learning will @@ -421,7 +421,7 @@ but this choice is almost arbitrary (though harmless). because it controls the number of updates done to your parameters. Training the same model for 10 epochs using a batch size of 1 yields completely different results compared to training for the same 10 epochs but with a batchsize of 20. Keep this in mind when - switching between batch sizes and be prepared to tweak all the other parameters acording + switching between batch sizes and be prepared to tweak all the other parameters according to the batch size used. All code-blocks above show pseudocode of how the algorithm looks like. Implementing such From 8819681562c539054c97097f6100d1a69bcbe75d Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 14:59:57 -0400 Subject: [PATCH 295/417] remove extra space --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 85111d11..06e2e88e 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -85,7 +85,7 @@ MNIST Dataset variables and access it based on the minibatch index, given a fixed and known batch size. The reason behind shared variables is related to using the GPU. There is a large overhead when copying data - into the GPU memory. If you would copy data on request ( each minibatch + into the GPU memory. If you would copy data on request (each minibatch individually when needed) as the code will do if you do not use shared variables, due to this overhead, the GPU code will not be much faster then the CPU code (maybe even slower). If you have your data in From 59667bd502e4ee05a5221293e4c2370bb065be52 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:01:42 -0400 Subject: [PATCH 296/417] remove extra space --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 06e2e88e..256ee07d 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -286,7 +286,7 @@ In this tutorial, :math:`f` is defined as: f(x) = {\rm argmax}_k P(Y=k | x, \theta) -In python, using Theano this can be written as : +In python, using Theano this can be written as: .. code-block:: python From 37048765dadf7146c3aafc4994cf8721cb7518b3 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:06:50 -0400 Subject: [PATCH 297/417] remove more spaces --- doc/gettingstarted.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 256ee07d..0019c3c6 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -331,7 +331,7 @@ The NLL of our classifier is a differentiable surrogate for the zero-one loss, and we use the gradient of this function over our training data as a supervised learning signal for deep learning of a classifier. -This can be computed using the following line of code : +This can be computed using the following line of code: .. code-block:: python @@ -357,7 +357,7 @@ algorithm in which we repeatedly make small steps downward on an error surface defined by a loss function of some parameters. For the purpose of ordinary gradient descent we consider that the training data is rolled into the loss function. Then the pseudocode of this -algorithm can be described as : +algorithm can be described as: .. code-block:: python @@ -425,7 +425,7 @@ but this choice is almost arbitrary (though harmless). to the batch size used. All code-blocks above show pseudocode of how the algorithm looks like. Implementing such -algorithm in Theano can be done as follows : +algorithm in Theano can be done as follows: .. code-block:: python From f78ba92c513edc177f1ff88eb34fb4a78310e652 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:14:38 -0400 Subject: [PATCH 298/417] extra space in logreg --- doc/logreg.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/logreg.txt b/doc/logreg.txt index c2979e63..b582acd4 100644 --- a/doc/logreg.txt +++ b/doc/logreg.txt @@ -246,7 +246,7 @@ within the DeepLearningTutorials folder: python code/logistic_sgd.py -The output one should expect is of the form : +The output one should expect is of the form: .. code-block:: bash From 1867a4e5a3f10730a6a844a91dc425962ab94fa5 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:16:05 -0400 Subject: [PATCH 299/417] remove spaces in mlp page --- doc/mlp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index 2a74aaad..0ecc7a89 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -178,13 +178,13 @@ The code below shows how this can be done, in a way which is analogous to our pr .. literalinclude:: ../code/mlp.py -The user can then run the code by calling : +The user can then run the code by calling: .. code-block:: bash python code/mlp.py -The output one should expect is of the form : +The output one should expect is of the form: .. code-block:: bash From d8294003cff53ea56f1d34c574f708e35ab63085 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:19:26 -0400 Subject: [PATCH 300/417] remove spaces in dA page --- doc/dA.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/dA.txt b/doc/dA.txt index 8ff26354..dd05acdf 100644 --- a/doc/dA.txt +++ b/doc/dA.txt @@ -6,7 +6,7 @@ Denoising Autoencoders (dA) .. note:: This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions - and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. + and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. .. _T.tanh: http://deeplearning.net/software/theano/tutorial/examples.html?highlight=tanh @@ -126,7 +126,7 @@ signal: :pyobject: dA.get_reconstructed_input And using these functions we can compute the cost and the updates of -one stochastic gradient descent step : +one stochastic gradient descent step: .. literalinclude:: ../code/dA.py :pyobject: dA.get_cost_updates @@ -209,7 +209,7 @@ need to do is to add a stochastic corruption step operating on the input. The in corrupted in many ways, but in this tutorial we will stick to the original corruption mechanism of randomly masking entries of the input by making them zero. The code below -does just that : +does just that: .. literalinclude:: ../code/dA.py :pyobject: dA.get_corrupted_input @@ -221,7 +221,7 @@ For this reason, the constructor of the ``dA`` also gets Theano variables pointing to the shared parameters. If those parameters are left to ``None``, new ones will be constructed. -The final denoising autoencoder class becomes : +The final denoising autoencoder class becomes: .. literalinclude:: ../code/dA.py :pyobject: dA @@ -254,7 +254,7 @@ constant (weights are converted to values between 0 and 1). To plot our filters we will need the help of ``tile_raster_images`` (see :ref:`how-to-plot`) so we urge the reader to study it. Also using the help of the Python Image Library, the following lines of code will -save the filters as an image : +save the filters as an image: .. literalinclude:: ../code/dA.py :start-after: start-snippet-4 @@ -264,20 +264,20 @@ save the filters as an image : Running the Code ++++++++++++++++ -To run the code : +To run the code: .. code-block:: bash python dA.py -The resulted filters when we do not use any noise are : +The resulted filters when we do not use any noise are: .. figure:: images/filters_corruption_0.png :align: center -The filters for 30 percent noise : +The filters for 30 percent noise: .. figure:: images/filters_corruption_30.png From 738b641bacd23511d0efdc87e9494f2ec8c1426e Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:21:31 -0400 Subject: [PATCH 301/417] remove space in rbm page --- doc/SdA.txt | 2 +- doc/rbm.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/SdA.txt b/doc/SdA.txt index 289a8b0a..6d9ba0da 100644 --- a/doc/SdA.txt +++ b/doc/SdA.txt @@ -6,7 +6,7 @@ Stacked Denoising Autoencoders (SdA) .. note:: This section assumes you have already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions - and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. + and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. .. _T.tanh: http://deeplearning.net/software/theano/tutorial/examples.html?highlight=tanh diff --git a/doc/rbm.txt b/doc/rbm.txt index a8079012..7a052cc6 100644 --- a/doc/rbm.txt +++ b/doc/rbm.txt @@ -7,7 +7,7 @@ Restricted Boltzmann Machines (RBM) .. note:: This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp`. Additionally it uses the following Theano functions - and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_ and `scan`_. If you intend to run the code on GPU also read `GPU`_. + and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_ and `scan`_. If you intend to run the code on GPU also read `GPU`_. .. _T.tanh: http://deeplearning.net/software/theano/tutorial/examples.html?highlight=tanh @@ -573,7 +573,7 @@ The output was the following: ... plotting sample 8 ... plotting sample 9 -The pictures below show the filters after 15 epochs : +The pictures below show the filters after 15 epochs: .. figure:: images/filters_at_epoch_14.png :align: center From ec4855a6a5eabdb5fdd0e8daf69218a21b2e5c17 Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:23:30 -0400 Subject: [PATCH 302/417] spaces on DBN page --- doc/DBN.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/DBN.txt b/doc/DBN.txt index bb0571eb..be7bfbdc 100644 --- a/doc/DBN.txt +++ b/doc/DBN.txt @@ -6,7 +6,7 @@ Deep Belief Networks .. note:: This section assumes the reader has already read through :doc:`logreg` and :doc:`mlp` and :doc:`rbm`. Additionally it uses the following Theano - functions and concepts : `T.tanh`_, `shared variables`_, `basic arithmetic + functions and concepts: `T.tanh`_, `shared variables`_, `basic arithmetic ops`_, `T.grad`_, `Random numbers`_, `floatX`_. If you intend to run the code on GPU also read `GPU`_. @@ -210,7 +210,7 @@ obtained over these sets. Putting it all together +++++++++++++++++++++++ -The few lines of code below constructs the deep belief network : +The few lines of code below constructs the deep belief network: .. literalinclude:: ../code/DBN.py :start-after: # numpy random generator From 85962ee63ae990e267e0875517de153e47cf777a Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 6 Jun 2017 15:51:15 -0400 Subject: [PATCH 303/417] spaces on lstm page --- doc/lstm.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/lstm.txt b/doc/lstm.txt index bde70bd8..aec230ab 100644 --- a/doc/lstm.txt +++ b/doc/lstm.txt @@ -75,10 +75,10 @@ previous state, as needed. .. figure:: images/lstm_memorycell.png :align: center - **Figure 1** : Illustration of an LSTM memory cell. + **Figure 1**: Illustration of an LSTM memory cell. The equations below describe how a layer of memory cells is updated at every -timestep :math:`t`. In these equations : +timestep :math:`t`. In these equations: * :math:`x_t` is the input to the memory cell layer at time :math:`t` * :math:`W_i`, :math:`W_f`, :math:`W_c`, :math:`W_o`, :math:`U_i`, @@ -89,7 +89,7 @@ timestep :math:`t`. In these equations : First, we compute the values for :math:`i_t`, the input gate, and :math:`\widetilde{C_t}` the candidate value for the states of the memory -cells at time :math:`t` : +cells at time :math:`t`: .. math:: :label: 1 @@ -102,7 +102,7 @@ cells at time :math:`t` : \widetilde{C_t} = tanh(W_c x_t + U_c h_{t-1} + b_c) Second, we compute the value for :math:`f_t`, the activation of the memory -cells' forget gates at time :math:`t` : +cells' forget gates at time :math:`t`: .. math:: :label: 3 @@ -111,7 +111,7 @@ cells' forget gates at time :math:`t` : Given the value of the input gate activation :math:`i_t`, the forget gate activation :math:`f_t` and the candidate state value :math:`\widetilde{C_t}`, -we can compute :math:`C_t` the memory cells' new state at time :math:`t` : +we can compute :math:`C_t` the memory cells' new state at time :math:`t`: .. math:: :label: 4 @@ -119,7 +119,7 @@ we can compute :math:`C_t` the memory cells' new state at time :math:`t` : C_t = i_t * \widetilde{C_t} + f_t * C_{t-1} With the new state of the memory cells, we can compute the value of their -output gates and, subsequently, their outputs : +output gates and, subsequently, their outputs: .. math:: :label: 5 @@ -139,7 +139,7 @@ In this variant, the activation of a cell’s output gate does not depend on the memory cell’s state :math:`C_t`. This allows us to perform part of the computation more efficiently (see the implementation note, below, for details). This means that, in the variant we have implemented, there is no -matrix :math:`V_o` and equation :eq:`5` is replaced by equation :eq:`5-alt` : +matrix :math:`V_o` and equation :eq:`5` is replaced by equation :eq:`5-alt`: .. math:: :label: 5-alt @@ -170,7 +170,7 @@ concatenating the four matrices :math:`W_*` into a single weight matrix :math:`W` and performing the same concatenation on the weight matrices :math:`U_*` to produce the matrix :math:`U` and the bias vectors :math:`b_*` to produce the vector :math:`b`. Then, the pre-nonlinearity activations can -be computed with : +be computed with: .. math:: @@ -187,11 +187,11 @@ Code - Citations - Contact Code ==== -The LSTM implementation can be found in the two following files : +The LSTM implementation can be found in the two following files: -* `lstm.py `_ : Main script. Defines and train the model. +* `lstm.py `_: Main script. Defines and train the model. -* `imdb.py `_ : Secondary script. Handles the loading and preprocessing of the IMDB dataset. +* `imdb.py `_: Secondary script. Handles the loading and preprocessing of the IMDB dataset. After downloading both scripts and putting both in the same folder, the user can run the code by calling: @@ -202,7 +202,7 @@ can run the code by calling: The script will automatically download the data and decompress it. -**Note** : The provided code supports the Stochastic Gradient Descent (SGD), +**Note**: The provided code supports the Stochastic Gradient Descent (SGD), AdaDelta and RMSProp optimization methods. You are advised to use AdaDelta or RMSProp because SGD appears to performs poorly on this task with this particular model. From bb2aa41171de24c48315578fd41f682e07284eca Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Wed, 7 Jun 2017 14:05:28 -0400 Subject: [PATCH 304/417] typo and space fix --- doc/mlp.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/mlp.txt b/doc/mlp.txt index 0ecc7a89..9e59ffbf 100644 --- a/doc/mlp.txt +++ b/doc/mlp.txt @@ -90,8 +90,8 @@ The set of parameters to learn is the set :math:`\theta = \{W^{(2)},b^{(2)},W^{(1)},b^{(1)}\}`. Obtaining the gradients :math:`\partial{\ell}/\partial{\theta}` can be achieved through the **backpropagation algorithm** (a special case of the chain-rule of derivation). -Thankfully, since Theano performs automatic differentation, we will not need to -cover this in the tutorial ! +Thankfully, since Theano performs automatic differentiation, we will not need to +cover this in the tutorial! Going from logistic regression to MLP From 8eb21daf92d48c020bfc7fe9b3ef680403e812ae Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Wed, 7 Jun 2017 14:09:03 -0400 Subject: [PATCH 305/417] typo on lenet page --- doc/lenet.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lenet.txt b/doc/lenet.txt index 09f50be6..84b7c3be 100644 --- a/doc/lenet.txt +++ b/doc/lenet.txt @@ -543,7 +543,7 @@ the task. Filter Shape ************ -Common filter shapes found in the litterature vary greatly, usually based on +Common filter shapes found in the literature vary greatly, usually based on the dataset. Best results on MNIST-sized images (28x28) are usually in the 5x5 range on the first layer, while natural image datasets (often with hundreds of pixels in each dimension) tend to use larger first-layer filters of shape 12x12 or 15x15. From 147cb2e9a9374d8f5b4673370c12ce6457b53cce Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Wed, 7 Jun 2017 14:12:57 -0400 Subject: [PATCH 306/417] typo/inconsistency in spelling of corruption --- code/dA.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/dA.py b/code/dA.py index 93a696f8..7d054b20 100644 --- a/code/dA.py +++ b/code/dA.py @@ -195,7 +195,7 @@ def __init__( def get_corrupted_input(self, input, corruption_level): """This function keeps ``1-corruption_level`` entries of the inputs the - same and zero-out randomly selected subset of size ``coruption_level`` + same and zero-out randomly selected subset of size ``corruption_level`` Note : first argument of theano.rng.binomial is the shape(size) of random numbers that it should produce second argument is the number of trials From 534e91585ebddd8238bf59d9cb9ba7fef2e6949c Mon Sep 17 00:00:00 2001 From: Philip Kirkbride Date: Tue, 20 Jun 2017 11:34:42 -0400 Subject: [PATCH 307/417] Add small note on easy download script I'm not sure the existence/option of downloading all the datasets via bash script will be obvious to people approaching the repo via the written tutorial. --- doc/gettingstarted.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 0019c3c6..f290305f 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -22,6 +22,11 @@ On each learning algorithm page, you will be able to download the corresponding git clone https://github.com/lisa-lab/DeepLearningTutorials.git +On linux systems, after cloning, all datasets can be downloaded at once with: + + cd DeeepLearningTutorials/data + ./download.sh + .. _datasets: From cb4261c830b39936aea224620c678480338ef272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Bastien?= Date: Tue, 20 Jun 2017 11:41:46 -0400 Subject: [PATCH 308/417] Tell that it work on Mac. --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index f290305f..7b1974ea 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -22,7 +22,7 @@ On each learning algorithm page, you will be able to download the corresponding git clone https://github.com/lisa-lab/DeepLearningTutorials.git -On linux systems, after cloning, all datasets can be downloaded at once with: +On Linux or Mac systems, after cloning, all datasets can be downloaded at once with: cd DeeepLearningTutorials/data ./download.sh From 36ec511d60746bde1d0e3905944760c92068675d Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Tue, 11 Jul 2017 10:30:57 -0400 Subject: [PATCH 309/417] add link to github --- doc/index.txt | 4 +++- doc/rnnrbm.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/index.txt b/doc/index.txt index 68a18ec5..e01e79fc 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -25,7 +25,9 @@ training them on a GPU. The algorithm tutorials have some prerequisites. You should know some python, and be familiar with numpy. Since this tutorial is about using Theano, you should read over the `Theano basic tutorial`_ first. Once you've done that, -read through our :ref:`gettingstarted` chapter -- it introduces the notation, and [downloadable] datasets used in the algorithm tutorials, and the way we do optimization by stochastic gradient descent. +read through our :ref:`gettingstarted` chapter -- it introduces the notation, and downloadable datasets used in the algorithm tutorials, and the way we do optimization by stochastic gradient descent. + +The code is available on the `Deep Learning Tutorial repositories `_. The purely supervised learning algorithms are meant to be read in order: diff --git a/doc/rnnrbm.txt b/doc/rnnrbm.txt index d64a0c4a..75e681f8 100644 --- a/doc/rnnrbm.txt +++ b/doc/rnnrbm.txt @@ -17,7 +17,7 @@ Modeling and generating sequences of polyphonic music with the RNN-RBM The script also assumes that the content of the `Nottingham Database of folk tunes `_ has been extracted in the ``../data`` directory. Alternative MIDI datasets are available `here `_. - Note that both dependencies above can be setup automatically by running the ``download.sh`` script in the ``../data`` directory. + Note that both dependencies above can be setup automatically by running the `download.sh `_ script in the ``../data`` directory of the `Deep Learning Tutorials repository `_. .. caution:: Need Theano 0.6 or more recent. From 81f257524079efc2c553beba0829c8a23d1a33d3 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Tue, 11 Jul 2017 10:31:26 -0400 Subject: [PATCH 310/417] typo --- doc/gettingstarted.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/gettingstarted.txt b/doc/gettingstarted.txt index 7b1974ea..99c7f054 100644 --- a/doc/gettingstarted.txt +++ b/doc/gettingstarted.txt @@ -24,7 +24,7 @@ On each learning algorithm page, you will be able to download the corresponding On Linux or Mac systems, after cloning, all datasets can be downloaded at once with: - cd DeeepLearningTutorials/data + cd DeepLearningTutorials/data ./download.sh From ebb8c21df3a3d073003e1323fead2150ada56ce1 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Wed, 26 Jul 2017 08:29:51 -0400 Subject: [PATCH 311/417] update nosetests command --- .jenkins/jenkins_buildbot_dlt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index c8be22b4..eb43d91c 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -79,4 +79,4 @@ THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} echo "==== Executing nosetests with mode=FAST_RUN,floatX=float32,device=cuda" NAME=dlt_float32_cuda FILE=${BUILDBOT_DIR}/${NAME}_tests.xml -PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/DeepLearningTutorials/code:${PYTHONPATH} THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32,device=cuda nosetests-2.7 test.py ${XUNIT}${FILE} ${SUITE}${NAME} +PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/DeepLearningTutorials/code:${PYTHONPATH} THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32,device=cuda ${NOSETESTS} test.py ${XUNIT}${FILE} ${SUITE}${NAME} From 8d25f1a91a656c5a0c67fe2434a5d37d89983665 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Wed, 26 Jul 2017 10:41:07 -0400 Subject: [PATCH 312/417] use nosetests directly for gpu --- .jenkins/jenkins_buildbot_dlt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index eb43d91c..8b57a1bc 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -79,4 +79,4 @@ THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} echo "==== Executing nosetests with mode=FAST_RUN,floatX=float32,device=cuda" NAME=dlt_float32_cuda FILE=${BUILDBOT_DIR}/${NAME}_tests.xml -PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/DeepLearningTutorials/code:${PYTHONPATH} THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32,device=cuda ${NOSETESTS} test.py ${XUNIT}${FILE} ${SUITE}${NAME} +PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/DeepLearningTutorials/code:${PYTHONPATH} THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32,device=cuda nosetests test.py ${XUNIT}${FILE} ${SUITE}${NAME} From 764cd4cdf5dc157a121a2fbffc2dec91c03f2ed9 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Thu, 7 Sep 2017 16:46:18 -0400 Subject: [PATCH 313/417] libgpuarray full checkout --- .jenkins/jenkins_buildbot_dlt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 8b57a1bc..15da288b 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -24,7 +24,7 @@ cd ${BUILDBOT_DIR} # Make fresh clone (with no history since we don't need it) rm -rf libgpuarray -git clone --depth 1 "https://github.com/Theano/libgpuarray.git" +git clone "https://github.com/Theano/libgpuarray.git" (cd libgpuarray && echo "libgpuarray commit" && git rev-parse HEAD) From 544c48cefbd9f149a0aaa4c8594b587fb502ff05 Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Mon, 30 Oct 2017 09:01:01 -0400 Subject: [PATCH 314/417] MKL settings --- .jenkins/jenkins_buildbot_dlt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 15da288b..818f79d4 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -5,6 +5,9 @@ export PATH=/usr/local/cuda/bin:$PATH export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH +# MKL +export MKL_THREADING_LAYER=GNU + BUILDBOT_DIR=$WORKSPACE/nightly_build mkdir -p ${BUILDBOT_DIR} From 057fd57b0e088090679a1b309aadf88ee89abb20 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Mon, 30 Oct 2017 09:30:30 -0400 Subject: [PATCH 315/417] Fix travis with newer MKL and Theano. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e2f2d530..ad729ced 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,6 +78,7 @@ script: - pwd - ls - export THEANO_FLAGS=warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise + - export MKL_THREADING_LAYER=GNU - python --version - nosetests -v $PART From 62e4c21c5607a259b2a394287e3b80a6af9de6dc Mon Sep 17 00:00:00 2001 From: Simon Lefrancois Date: Mon, 30 Oct 2017 11:47:23 -0400 Subject: [PATCH 316/417] set OMP_NUM_THREADS --- .jenkins/jenkins_buildbot_dlt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.jenkins/jenkins_buildbot_dlt.sh b/.jenkins/jenkins_buildbot_dlt.sh index 818f79d4..fadd9f9d 100755 --- a/.jenkins/jenkins_buildbot_dlt.sh +++ b/.jenkins/jenkins_buildbot_dlt.sh @@ -8,6 +8,9 @@ export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH # MKL export MKL_THREADING_LAYER=GNU +# Set OpenMP threads for stability of speedtests +export OMP_NUM_THREADS=1 + BUILDBOT_DIR=$WORKSPACE/nightly_build mkdir -p ${BUILDBOT_DIR} From 212a8cb04184adc28f73f03c363da3401c030d36 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 11:45:25 -0400 Subject: [PATCH 317/417] first commit --- doc/contents.txt | 4 ++ doc/fcn_2D_segm.txt | 165 ++++++++++++++++++++++++++++++++++++++++++++ doc/unet.txt | 139 +++++++++++++++++++++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 doc/fcn_2D_segm.txt create mode 100644 doc/unet.txt diff --git a/doc/contents.txt b/doc/contents.txt index 3b7a16eb..06713b3a 100644 --- a/doc/contents.txt +++ b/doc/contents.txt @@ -24,3 +24,7 @@ Contents rnnrbm utilities references + fcn_1D_segm + fcn_2D_segm + cnn_2D_classif + unet diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt new file mode 100644 index 00000000..d03b5cf4 --- /dev/null +++ b/doc/fcn_2D_segm.txt @@ -0,0 +1,165 @@ +.. _fcn2Dsegm: + +Fully Convolutional Networks (FCN) for 2D segmentation +****************************************************** + +Summary ++++++++ + +Segmentation task is different from classification task because it require predicting +a class for each pixel of the input image, instead of only 1 class for the whole input. +Classification needs to understand *what* is in the input (namely, the context). However, +in order to predict what is in the input for each pixel, segmentation needs to recover +*what* is in the input, and *where*. + +.. figure:: images/cat_segmentation.png + :align: center + :scale: 35% + + **Figure 1** : Segmentation network + +TODO : reference de l'image + +The **fully convolutional** network (FCN) owes its name to its architecture that +have only locally connected layers, such as convolution, pooling, upsampling and +no dense layer. It reduce the number of parameters and computation time. To obtain +its segmentation map (output), segmentation networks usually have 2 parts : + +* Convolution path : extract semantic/context information +* Deconvolution path : recover spatial information + +The **convolution path** is used to figure out and interpret the context, while the +**deconvolution path** is used to retrieve *where* in the image were detected the things +detected by the convolution path. Furthermore, to fully recover the spatial +information lost in the pooling or downsampling layers, we often use skip connections. + +A skip connection is a connection that skips a least one layer. Here, it +is often used to transfer local information by concatenating or summing feature +maps from the convolution path +with feature maps from the deconvolution path. It helps combining context +information with spatial information. + + +Data +++++ + +Polyps + + + +Model ++++++ + +The architecture for FCN network depends on the precision desired. The Figures +below show 3 different architectures : FCN32, FCN16 and FCN8. The convolutional +layers are represented as vertical lines between the pooling layers. +Those pooling layers explicitely show the relative size of the feature maps. + +.. figure:: images/fcn.png + :align: center + :scale: 50% + + **Figure 2** : FCN architecture + +**Difference between those 3 architectures** + +These 3 different architectures differ in the stride for the last convolution, +and in the skip connections used to obtain their segmentation map, as you can +see in the image below. I will use the name *convolution path* for the network +up to *pool5*. Note that these 3 architectures have the same convolution path, +but their respective deconvolution path differ. + + +1. **FCN-32** : Directly produce the segmentation map from *pool5* by using a +deconvolution layer with stride 32. + +2. **FCN-16** : Sum the 2x upsampled prediction from *pool5* with *pool4* to further +produce the segmentation map using a deconvolution layer with stride 16. + +3. **FCN-8** : Sum the feature map obtained by summing *pool4* with the upsampled +*pool5* with *pool3*, and use a deconvolution with stride 8 on that feature map +to obtain the segmentation map. + + +.. figure:: images/fcn_schema.png + :align: center + :scale: 65% + + **Figure 3** : FCN architecture + +As explained above, the deconvolution path is different, since it uses different +skip connection layers and different stride for the last convolution. It thus +yield different segmentation, as you can see in Figure 4 below. Combining layers +that have different precision helps retrieving fine and spatial information, as +well as coarse and context information. + + + +.. figure:: images/fcn32_16_8.png + :align: center + :scale: 30% + + **Figure 4** : FCN results + +Note that the FCN-8 architecture was used on the polyps dataset, +because it produces more precise segmentation map. + + +Metrics +======= + +1. Per pixel accuracy + +2. Jaccard (Intersection over Union) + +More structured + + +Code - Citations - Contact +++++++++++++++++++++++++++ + +Code +==== + +The FCN8 implementation can be found in the following file: + +* `fcn8.py `_ : Defines the model. +* `train_fcn8.py `_ : Training loop. + + +TODO : import model_helpers, dataset_loader, metrics + + + +Papers +====== + +If you use this tutorial, please cite the following papers. + +Fully Convolutional Networks for Semantic Segmentation + +* `[pdf] `__ reference + +A Benchmark for Endoluminal Scene Segmentation of Colonoscopy Images + +* `[pdf] `__ reference + +Papers related to Theano: + +* `[pdf] `__ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. + +* `[pdf] `__ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. + +Thank you! + +Contact +======= + +Please email + +References +++++++++++ + +* ref1 + +* ref2 diff --git a/doc/unet.txt b/doc/unet.txt new file mode 100644 index 00000000..22a2e17f --- /dev/null +++ b/doc/unet.txt @@ -0,0 +1,139 @@ +.. _unet: + +U-Net +********************************************** + +Summary ++++++++ + +This tutorial provides a brief explanation of the U-Net architecture as well as a way to implement +it using Theano and Lasagne. U-Net is a fully convolution network (FCN) that does image segmentation. +Its goal is then to predict each pixel's class. +Compared to image classification, the difficulty arise from the fact that localisation is important. +The network must capture the overall context and recognize very precise details. + +Data +++++ + +??? + +Model ++++++ + +Compared to Fully Convolutional Network, the main difference is the added skip connections between +the contracting path and the expansive path. These skip connections intend to provide local information +to the global information. + +U-Net architecture is separated is 2 parts: + +- 1 : The contracting path +- 2 : The expansive path + +.. figure:: images/unet.jpg + :align: center + :scale: 60% + + **Figure 1** : Illustration of U-Net architecture. + + +Contracting/downsampling path +============================= + +The contracting path is composed of 4 blocks. Each block is composed of two 3x3 convolutions, activation function ReLu and 2x2 max pooling. +The purpose of this contracting path is to capture the context of the image in order to be able to do segmentation. + + +Bottleneck path +=============== + +This part of the network is between contracting and downsampling path. +The bottleneck path in simple a dropout layer, followed by a pool layer and 2 convolutional layers and an other dropout layer. + + +Expansive/upsampling path +========================= + +The expansive path is also composed of 4 blocks. Each of these blocks is composed of a deconvolution layer, followed by the concatenation with the corresponding +cropped feature map from the contracting path, and then two 3x3 convolutions and ReLU activation. +The purpose of this expansive path is to enable precise localization combined with context from the contracting path. + +Advantages +========== + +- The U-Net combines the location information from the downsampling path with the context information in the upsampling path to finally obtain a general information combining localisation and context, which is necessary to predict a good segmentation map. +- No dense layer, so images of different sizes can be used as input (since the only parameters to learn on convolution layers are the kernel, and the size of the kernel is independant from input image' size). + + +Code - Citations - Contact +++++++++++++++++++++++++++ + +Code +==== + +The U-Net implementation can be found in the following file: + +* `unet.py <../code/unet.py>`_ : Main script. Defines the model. + +The user can now build a U-Net with a specified number of input channels and number of classes. + +First include the Lasagna layer needed to define the U-Net architecture : + +.. literalinclude:: ../code/unet.py + :start-after: start-snippet-1 + :end-before: end-snippet-1 + +Our net variable will be a dictionary containing the layers' name as key and the layer instance as value. +This is needed to be able to concatenate the feature maps from the contracting to expansive path. +For example, the first block of conv+conv+block, including the input layer, would be : + +.. literalinclude:: ../code/unet.py + :start-after: start-snippet-2 + :end-before: end-snippet-2 + +Every layer will be added the same way, by specifying its name and +input layer (or input layers, for concatenation). For example, the last concatenation for this +architecture takes as input the 'conv1_2' output and the previous layer as follow: + +.. literalinclude:: ../code/unet.py + :start-after: start-snippet-3 + :end-before: end-snippet-3 + + +To build this network, simply specify the number of input channels and number of classes as follow: + +.. literalinclude:: ../code/unet.py + :start-after: start-snippet-4 + :end-before: end-snippet-4 + +Papers +====== + +If you use this tutorial, please cite the following papers. + +U_Net: Convolutional Networks for Biomedical Image Segmentation + +* `[pdf] `__ reference + +Fully Convolutional Networks for Semantic Segmentation + +* `[pdf] `__ reference + +Papers related to Theano: + +* `[pdf] `__ Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Bergstra, James, Goodfellow, Ian, Bergeron, Arnaud, Bouchard, Nicolas, and Bengio, Yoshua. Theano: new features and speed improvements. NIPS Workshop on Deep Learning and Unsupervised Feature Learning, 2012. + +* `[pdf] `__ Bergstra, James, Breuleux, Olivier, Bastien, Frédéric, Lamblin, Pascal, Pascanu, Razvan, Desjardins, Guillaume, Turian, Joseph, Warde-Farley, David, and Bengio, Yoshua. Theano: a CPU and GPU math expression compiler. In Proceedings of the Python for Scientific Computing Conference (SciPy), June 2010. + +Thank you! + +Contact +======= + +Please email + +References +++++++++++ + +* ref1 + +* ref2 From 46f6a356cc6ad3e601c25c4fff87192606389a0e Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 11:46:16 -0400 Subject: [PATCH 318/417] images for fcn and unet --- doc/images/cat_segmentation.png | Bin 0 -> 359051 bytes doc/images/fcn.png | Bin 0 -> 51573 bytes doc/images/fcn32_16_8.png | Bin 0 -> 109595 bytes doc/images/fcn_schema.png | Bin 0 -> 22929 bytes doc/images/unet.jpg | Bin 0 -> 41960 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/images/cat_segmentation.png create mode 100644 doc/images/fcn.png create mode 100644 doc/images/fcn32_16_8.png create mode 100644 doc/images/fcn_schema.png create mode 100644 doc/images/unet.jpg diff --git a/doc/images/cat_segmentation.png b/doc/images/cat_segmentation.png new file mode 100644 index 0000000000000000000000000000000000000000..490a211890d12ea963ff7af214a7805873171b66 GIT binary patch literal 359051 zcmYIw1z1*5v-LqxLJ&}p6ch;w5lLx9y1PTV1f)AeMCp<)Y3c4#0g-NyR_Sh#{)g}0 z@4xT!h=RO_bM~G+v(}n51jx&Z;b4(sAqavaAug*GISK9O1`=&wsT8Fut`s|SAaq|+Lee-N;))%s{+tUlG=%bWZp zURfw@@!0FkcC*pceRgj(WpC=plhSv|^*P1^KRhA$(9c^`RD|~8^>kfy`lCO|=(rT@ zHZy!|YCUQ{KKkm?4I% zKi<>xn1uyz4a@T-wJOTh)!|2^T|P@$8Y|GS--x9Q4Pf3>}L5#7*=>_w{U3qhuDt+V!->?)Z1HcmBIN-u;!9V?0jl z|9*da*WmoUrNuR+SgX3IvC;EwqH>XTynY-!(Y3EbIdY7u1-}Y3*a|vW{q*BB%1w-o zj2t$nYNd(L-2d;Xp8R$7nyRrSAt5O@>6xgu(aR7GL44+F;H8u!OIZ46z9UMj(t?Ib zE`fdJ+y(FCLM2zx7~TO_f98GR5zmZaQv1IP`(U5);P*Gw2yr8Gb4)}+LZYgwDj*;L zxgRY#_xEpiPtVOs7G~z&A3xyP(UJA_bt)<;JZ02tks0ohi7ilS&#)`%>=lIuVqXefaPpC@83;;q^Fc zt@ccl=P5*v4;DIm+^8_csxV)oPz$L zp*we3Sy^vU3h){J2>$u=Co4e@%Kt45L$TA=jFq`Lx7V4AzrNf4@<6Sl_4&1wLlgDvOAt{$NpYx`+{_)xIUcPD}!>-Yveu+ zjq%E0R#(j9JGzPlm0ss3a1s|MwR48JpFI;67Djx;sRKhpt@}TJJ7s)BK9(sF6&DwW z|4@`kuZgd?9-ZM1D;pc!SVCf=rM30N$$=$;7t}T|FmSLob~&tb{rdF~a&E<3K?_zM)^un4 zPfJfC*W|BX>RMW+rlzPqg@qfi-S7AJ+A@_H;1qt5f8-adlr62%;IgB}h;i%xPGhp$ z-l7cbv-8G8CB!b|G(qo+hG$yS4X*Y-f=TQbyXjdHP?40Blr2G*z7&35YHI3Po-I1s zu#k{sR^vd`d>kActI>k^#6+^bJ;BQ}Hc|UdDv0=Wkst{1qxJEj@81ok>l|HsOb*w_ zuUkjLD?a}C&uMOMuIm6fA}%R;l5qXo{~g&TrFy3gQg$=>nXW>u>RV)->ls=Ho71OL zwf4ojjSL3V1busrZU;ONw%cWr(F|b}{E=M?$D?reoG*ip5}N01K)tzM`Tc-0k*a57{5%H*dDX zAH3Wxj7my+ZEjAF-Eht4?qf}~>(@u}RBk?zag9t!u&}a9j*G)U?%usyT3UMJ#*M9F zJQ9*|IQx)8F3$G)j;nfp{#38ETY!?Gt*z~LuzJ8DNA^Mcf0H}jiR}2~B~X+zF_^UZVfEt~kqP z3~ci~>6SQCfSXI%Ao?Ruv|Xw_-K*6O##+_LTmq?+u?dVR0wq@ z$Ze*?w6i%lWSj(dAzkw8fA=%#Ng!jO@8GR&p^5(|WlN|X++0yZ4 z*3&$J=_f~9dT>w0@`ly$x#vkm$=UwOkli=~)VaYdDfqvI#l=wwuD;2DjO3DeJxj;Bwb9yjpQ(b2iPy58fqqvqkM%M83)sQ%^xO*Vmrj=fMFol=JX zl%sy{*TZPa#I%neTXjUzgocKumi8I8hq}7CEiNcA(m$KW zda&;uwB+P^uu9JuSwir>^6h_ayNQvn?+O3`S`lJLNA5DIzd2YP@yJ&x)}8L{l~r$Y z-kPZ}9}=iDeGsMsz)f;%UL)pwE=+KJV}p{GGSofr) zB-gz~&%qf2thO%-8FkKEx_<}Azxvz~5-*{7^LCciR<1VNlxVX5!_dAUl z1A~LHF)_!>jDzy?*A{ycQLkOQ!jm+-1)7zsP;2=4`QLvsl}i-}2?!wJaae}@iH(C3 z)SPNLRx~~_Vg0Kl?Gx62(x`Nz;P${AF(SWst^GDb;SHC&`PcuGPog=aB`a;`evq)4 z5SSVh5yb*j5p+LXhX8|Rx=mZ`xjWy{8gOT?aJ<>u+thTaAr?SCK0f~Yjl;GeBFoW& z(lDBoi*Qp6#nug{assJFm#l=OaVF0d@Qd3if(jJXxD1VVplS$-+mylYrf!pr9 z1dt9`q{5--%^Gw-Z~L>AUu~Yxhs+_*-|)zGG1z@#$~P3|Ys+&dJ0PMxZMF<%et;8HQYep&kT>1kPhmSQcDtTr@&A>`7;sn}HV2SQ z8e-BN2MO!xccZ?AzL>{aUS~V)aGZ>djUmrjuqXWb^-Cg(7LcP36wt|(NeEplE34*S z^HHjA9c~z9^#RD+SR@<`0%KF><{kju&sS*Q8*?{7+)-iP=QgDP`&9%a>z?T1UXiZgZE;&l|zX((^i|;o>@4Q@##-^Q2Fs-dNeJRW)Vl znPlf%VPPdvH&Id?seUwi8bb2a!Z|{6(f=U{{c3}a?#{?Bpc8;M-=oh7c%HbpxoK)> zh=_=&Kn#`Oc>nwJ0jLs27o4bD7#KgReYKgd3KA67v$L}s*u?UH*#M9LcJ=f;2dLZ( zu>a5Z_cW7SbhNZTdV8(mk^dLp^Se8lhq>Io9aU{P#tgBdR;=s1IkhrTY3XXk0JY%~ z60+0!uRn{6wEHf3@@Ycy^8Ju`{Rjb=w}(;cCF#f{T%iJUKp3j3szQ;wPi)tJJ+-u4 z14{1QGjV_SJqy0j!sI{m#=n8WH|bak1>& zy{_))uys`k!&9p+miL2GLPoroMYWa;X+G)$zs}-%=PgdNegp{)4puEt2kp39UL0o{JDno zvnPN0q%2I1hp>}|06Wxmt4!f{_i|zqO_M*nINcKZvH$lYkQ!K#0A&n-R?E$Qw|a(h zdNzwj^6c%{%p|)%E)@7jM_`-Pbayjd&+nDbk1C7KUE$V@Bno;|@YJurIy*?P#^b0( zY-c6T85f?t7Pb@XCV)M-RW5e+;Ex|4natkxz0YYqWoB)C3#qNGg$H`P5pQ;LA96V` z3b|ySk)J=8e?~kc<9z)OYaj`L!)URd;DZNgaCQL&1G`w@cv^d(ot^#ZQw4@-FLmj_ zbSSUXw=seE*DWtEhjAtz>3JP)SA6;MMKpwzP%uHMjOFvYe(&$U%o{faKVCy#JyLC# zZ1pGUF!vLE{E%BjH8gargq_-ykxwm+UTUSgmc@(V-XtX>6~c&yx~AiPx$qJHVFHJx z>RAtvsyMD+46LknP>=QY=3P^8v9QWYOKTlg6puEi0oqPO?Hv>_la!PM7I!5v?43ci zg$%J@ctYbTBZQs=83lK#?}iVh*FSw=?KaJY<6$0L0O~DAne;W}a!UyTYw8Tsl5NqMkcf{wQ`SxJ*NC%<8!AY>pwbIfTX|1ioh&)3y5DfY^@AX^! zF>hWm?&Xq!? z*a`ADSRVZLxjDOPn0gm6f1qTVEYr?#!r)@8I5n5k_xH|_T?I|4;6aAxl9*Nh$JV=z4bf!r#geoKe)1%D@$lZP4(IANx&mOETu=}m)`f5lsko_Tt{KE?906gi} zNy1?<446eQmr+SWElZM6Uw`Ls8}Mt&qWi(qwDTy)p`ux-R7_Kjl&qBfwra-DjkMz{R!XJ~nGsmXgvw?a zsnjwMipbvx7B83sT`n!lrQ7a(CpPo(Kbv!+b~BWMY|6=VcY8xD#BP(KnHi0cm4Z*a z?Ut$D`U4u7B7J7QK->VqjJP(nky$RCjBP=9E(0g-k z&VGMcfs3meK;@@@=8VyuvtFl#06L2Gnhy^S8a(B? zVoBov%|h7t-O0D?l@}6-y#CinOqqk(>46aoT`l#z;R*#B>9Bnc1yLMyISS1=Dd&v3 zX7|&F;T5M@<;NfFEACXNpIqikN)hkYo0+-n;0rC=r-)1JUeT%Ac>i1XCLpu^$;gt5 z)va?_X^~m%OWoSs)crk^z-eV+WAlWGjEoGThzsy?e}9Ql-!UvHoEn`*H%EXe08W;A zleH$e`pDbQ0ex9bR`qmtt`20h1FiKF-n!&~;$Bu&bvFhVC>+oTyQ$0i70_xS$6x73 zfY7sS_piwIdF#ALOQN=4XX5*Fx1cCT9#;WS;EKC$Xz0 z-b|v_DO54F9|=CFlnfliCy+Cx7q3;Oi|uTqdOAwYaL3^?X~vxl`TF_1UzOH1?BI5y zd3$K5T54w{+0%g+l=&-`v)aT7WFCK*3975NT(HWDbnkUzVT^_^@-NAiIBL5o-MnwZ z;c0Dp=RfJnAM&e_$7?t}QH7+AVYz zhj*?*uL1I!_rYqkG6Sq|Wo)e4v8=Jm@Z%dYTT`{=B0+b*H;6-N@%*>_2dEBk8ds16 zp(u{zDw)DYP)jW4e~qG(6Y##!PLm&oDE7X*Nag7p$f2dZbR+cjmd9h@8a?OU+J5U< zm^Csy?vX8-{K8Jdu287!4qw;L+@y7F{PsYfP8H+Tu2zzG9>T5xO(CkM>D&>_O7R)L z?LK)BN*biwh@-qc2tr zJ9LeAHk@#RIlyCL>9w+3wCX)lmD~C+Ut*+{-4_t>0%8rR8j3i92t7d{5Rwmme)EZ8 zqyM8h0Uh~H3#b=FZ%^;{pp8kV3U~sf2bA00<$Mc4K|z_BnIRA<)ULIIFVj#Qd<+Yt zHm6c+vOZ&A^a zD?q`{PJl2#AHp2ur#rL_f`Sg}>c1jsq-2wx?OaeS_9m0z;mMy#8!0F#K=978M62Mx z`AI0W(8@F$;k+e+>LVm%;c{`2IMz49o!d#CEZenPAlklK(V)$s_QeGe%1?9T3H&Y0 z`UAtk0JE@t?c zisE#W7Ufbge$O|5e>@?*z@_7>%C5e1)k{d|F#uU%b2F`RqX`kkabv z>JoBd;u?qp4@Ov_mS4YIfTnLXyUzW<0nr3B>r%t^nAhwdmX0CxxL*HaVPYzP5QAm8 z0O(aw!LFq>2|QwtPY^%}^fwII#a`*_m%;^?m(75-0lon;oFd@K^Yp1LP;l3QBv2P% zIiWeVd2SZU(yR!W1_a2kd%TAOsGhh19YOwFG#JsYY;JKVADRH3muttMyTNG2q5I!xfiT`a> zS(&K&-vdQ3JRgn^XyH=(M3kFa8Wv#t;V}?-oga)rMhC^#74&op3JQRs04TnTg@Fp` zP5OyXDuxNpb4m#xz={WYjuZjHyQk3V<8#?Q=Zbx-br&B$qFO~lLQaknRFC}peAWDO zNMb@lLeP1Vs{i4%Ii&$Y8$>4vwv1F%1|V5qMzqeijn&xB^(OO@{YM!c8yMI*KjD7! z+b24Dd0#)oF_4orHy#V8F7c8PoBCBk4fy2ordbo6?1M95=}?WtlrGbhP0%-&m9BwY}}5a zvb2$FQq+Vv^W&1COTq*a`qNZ?O(ETuTDO`}qBc_=Grlsu@7M3V@csQwAQB)`je-Iw zD9xfA2iO4G2u?%UGI!`?3F^d$;p9MO|DMz6e#HDnRR4i-^!BHegajUsf7>2n>-}jh zkips5*!E^3B(ta#FfSQgVIll+$Tnvh>w#Yb`~YF%1mH)#!wM@@=1NU|gVvA0*cWB? zf_Ke91%)4pdynVSk*um}OntpO$nPMxjf+)NoICYCI(s3i%+z8)Lh#o(amYxD>o zTGmt)&oc(MEq;#_u!>zL^_7>(@Lq?hWU#t0+dou9 zXql|zBlfd|YuWt#31s^JSvG25oSpMXA8|;V|($dxi~Nh)cONYLeTx z%K&gf+W?l6u&VHXBvro!NYRUAuW06r7#g<&C9@J(^nEij426WgLsfoAMRg@oO+=GZ zP*|IoIKhzu1_g~_7vKeY`hyE)`Ow9I(2IKY>T^hl41Cz{qQHdmO*P+hgF z)Z*4N-`k9inhE$4#0gZcm5S&VF~i$@#Zf#`6?ZIi0!0==`D^ zbwc~Snl48&(EZcr-0hditNsr7lMOFF_N;N`bqF@vKGE3w)1H|r`A$@DISNVoyRf14 z#BSG7kj{9dz=~=ALWfGnqC2f{@-Q{9=HOo{JB_)S?(|@lPMFub8$4QMM2Ccz-0vAa z(Qb%L33k-_DU47ubstFnEuwnN_uGXu?(*-<{+vdJx`^9gA{U8rWgW*%aU8CsI18=d z_(=ZtWj(j!YI%EKRCL9Q=!DArySlx7)@G}%jHhR)N2!f=ODk&*gb&z?b~CIPms0up z-8l-LNBuKmqZDyqc&m zCuC6m*|jsjwWSAgsRw3>q~z<@>(G4ywb9VroP>a27$n`nBHhqA(J+ce^C8-lrFzG; z#jUN&-1xYQM9$a0`XLPf)_|k>WJ^X#X$bxS{krn6G8n!rY;00eQe9kRTnM?K*VNF^ z9+W2I>@z;0F9o&^7H3Lx6M_>L@?BM$i2_bKjT1QcW-XVW>aEMj2wD4jYtz#m``)jyO1;^XMz78B=G6Mh=QhQE*u2OZ z-Rh1yB~sRi&gxohA}_!1C=kgS@MmnP{~#F)lPxmXIjx*Js@AusSifj&WSQ5wY#-&n zfkVmNX(@hmH^vS@2GiFHGIsLL&?WhonR=^d@4m2cN$IO4nxD?ztv|{JrN@(mTn)l-~bD~0W8P|~M;*?X&;vgFEndXYlX zNOocQC{|x5$Cj)(dsXsFTJ9zH=42%g3iA2LP>V>dc;F%biF3h|ea=Wu@=b5LdbHvn zIrj=%85Z?I9-La2$dR!jNceew!|AKO0mJs-ee#L1T9tR5*go?)FDx?#Z(4|_s;gk6 z*<5ZUM(lXmM1+Tj|9!1esL6r+0#l1AC6|D}%~+DZbUuH)s;8nI{A}Ci|_5hPC)P>4XiUVhZdpf{Z@- z(b6X@VfOD-_(v$BePK`X&rrWL6qB(KtVlAza_Kf zGB?v3bbnX;ZcX7Z z|D!HDmzs{*qp$OLQnv7(?dreEme`9;jrfyv{~yL;#p`3{24*}z*XhtU)PJ?JZUyH4 zJ(O`EsSx1&S}x+DYV_0QfF{+EHL??JO`J1MYoK4exO??&gN`3 zyU)%4I5{xmFi$cp+L=aum#?^eoh}UToWiZDj>CjDnDGq)>|s05|j>7sw2w`=G>g zhq`;C%|fEo(b1t4@K8ToC3x6_5luKVSKp^`ycdVr+vvt&w{9|I^`?utWun5q)Omff zO6-#J+`o$QS$C~mv#gbS)w)1`+R|@X>x>gO*8_?_P{$NdVHXv5k5ci%kr_l#j6^5GF zAMS#cS$YW*;nxs}e8D19!e0x1GL0-*53VDBV}fk%4s!^5dP}raPt$Rx9$Gzmx<&a< zxoywnf+Iu8MyNBiej1)^-k29pECQmoWpN5R_GkzxW+dTO8jS%Je z@M~EY&5QEhrhi@a=%*99Vz)0FYd2*+e(0}!Ic#!(>X;UNwDNwiiD%ua2SJvr;|Whz zTXdELrWU5vB@;)(Jydb;9Ud^7(m(H!xZK=V>`$o0MLi{L5i9c4#S}?Cnh~5=c*nU> zch=clpHxEMKINC{DKjmb?Wyomsr~vs0ke7X&JxzLOor4iuAc;mPf5JH(y^;rhF_>E zD`87^%DGIOjIRYdBUOga#mg`?E^ji+*JT&yq8G+0@9`L{($dnPEk#q7-ymW70f$WW ztgu4U>g_$?N!;aT1L+`JgI7tTP&L-&oAdgw2d$(@$;o$a-)<68f%45|F^qv=_(Go= zVi^Mkdd)N6zP+WEK>h(m1(ihQuNWo<2I2$iOS)RU{Zgnf)ee|0PD3(= zdQt^18BOsR!3^_^o>j>BYi}~+;>oSf-j4oV0y#WFnfbdT45}<6hSaz5AE&9OD3y~w z?yy*45-HLw95jC+EtSca^~7M`SkXW7qgg=cJMGF)ypAk`0Cl7L>8(_e59s}=jM2W8 zkoTcx8Dru|<#CV)MJ<1}dnYAueCpFy%EZz`gf3k&^kh3IjNK`&+)C9GacmtOLJFvT-1&eLk~d{J;9G@_cz4-2{UqD8FA~MPS4hp^F9o{B zcHRdt$S$GD+UWoL!~KXtui=Pu$Z|q(Wg;?`IISkYrg7yaDjP2V6W0w~3MT3pO=yk*z=LM3_C#f) zX&;5X{T^5$0EcaFZ-X85ZrnzGK|ORwK?uEa%lvJ-nX-iP8{PsSMnq`#CR4$AwOM?<8`}s zqyJ(A1$plJ;J3Eo#iiPbYUlw8;*(^LqEAnGQ7je4y(2!{>Hek>H)sCA0>|pS%x1dY zxN^eQT2j%Bs}D<1PkxZv?VF8a$C*OL+6hnR!?c)?Ve)2u%@<)?Hhn!`M&=1$$_UPB zmImZ-e#lMh4M!0qG!4gj%ypsui>}yj;!AVim{yJXClwUlm@z9;nj6T^TCxM7gIb)& z8^OgkaiqvklRMs4y5gAuMFaN2WcY|t>eL%f?{|+zM;_&7c)GSdN0g__Y)|);ksNm0mP7x^5j%WXN{$5;{mg2QvlzvOO z@LmMWzhDp2NopJc%Puc3FB8*xKH(Yk*uXgi?l@Dhg%}zbfNM!)=-9-c zRr4Dg8o&!V0+u|x@h{MPSorq_lfe8!Mu7-8!l zqND$fnP5Qd|G+FE+s_+YrXRcGqa$DLsIBI$XB12QFBV>&yk8)rKt-N6T00R2YDcqL zjn;A^2wCOdzU8U!Y8AF?x_dh9+MSsrTQ7)gek4sUkH~Mm+ft6NTze&jAoA~zQ#)q+ z?sFR)Ii?K>Z65pF>^M%<7k#9ZEVpi z1rZl7)l4IzC9#|^Ldm_amq*PhsEE%&oZfn1?`KIC`1!TnO+0_RHEs&xoYz|K(8?TV zPFiK8DBT!gc=oxSSdDP;tiv_xAYQQF7PI~AFzXufyybGiJp^%d__?v( z*GNj^R?$}wS7G19DS(0?=44#xY-QnJx=W`2d<$AC;&C%~M9)yng}ALdC4k)0GOzFwMf0 zXSN3Iq4%Iw&*6Ep4+f5gH!oKLf`i$iA>yZzDgqqmW)?_Y<%|j0?HVaOLkHj(0BRx`MZP- z?5-Mew6*&bb^1fMLgyG%^MyJ^NPUe_S`E1ABQtZANw8>Sq%Z=N%Sdnid>tK_jVUT^ zt6Akc_3OC>24B|VGf^s{2v+}3@}J_6Kw6AYW7o*EwoLKXkHq*bEbKBv*fQblFZ%gD zx%-qg1Um2;j_|~4z=p66;_r>3} z4>_@idAj{CpGsi1J)Zg30m0$OyWMB@2hH+eSy@RLYfSR!@2^FSd+NArrx`W#N4@N; zBbg;Wy1HY;p>u-vHRcxo4#m^dVr@|^%c;^esLJc^Dk47nr$%c&}|1k?|rgw?&MSj=G?Bb<4de?BVCI<@Cp@Z*Oon7$|wdU{gGt-FlfWpqXpw& zp1SHlK<^N?E@qvj=vv4<-YT$AgUSkIu-5&^+IF`oRtFj;SKcfz9PPc=xkdgAy!$Bk zxwxE{`_pJxrvmO{fSC~a1*Ss~O+l%BDEtYs4+pd{p{FSaIw@$Ya1RX*E0L9z2M}jn zVl;*^jDAgGg^4^g**VK3q<7eRmL-&5(>{z)tR>I6eowwbV};fCZn=?(wY6-FBJFK@ z4#`rAm*K&st-`V^5%(2yIT}6^1RP-E6b_iH`x^RCVFC+`GJwc~kLCkw?({amWPfL&f-d1E|Rit?D{k?Fz9V^S-_K*71B=Lyc;oYnV zM{y$W;Y(8VkSX^uhLB0~cdOBvS!4YKw)jUgp%Rk3f*E0ZW#2D-R~6Hvf!;^Q6*2x3 zST|J55Dq(ZCCqU1I{q8s9&gmxm`rBdn+A4SNKR81u2Bd&m_Gotz{?Zr$Ul za9mTx#SO#d`s3;~kS+p_l+K`k+D#tgeSJ2apXiTA$w8WY^QNR^eQj+LY@7O|zlARM z)*L_<1?t4mxNApV4Ynq*a++&5+S?=Jbz(w3J|CDG5*u7_FX1p(sO!JHwR|ep=kXcx_!ZJExpsu4x-z@i9rDyNf#TuBYb$%?-*$O$+Jnq6 z`xGP#W9|ezD~zEw4~)Sl7kf&8T<4P~kYK|``tE$aLoM-?`UN7RNG*eaeSU%JbJX^4 z9aGx~>HPr~8k)$)Aesi(@GriUSe;3QdRc#S(8yaIPjJc)likRkli|NvjZRd(e#*YboLO zM~X8uifn{=${ZYM3prDr-kI{XPYh;gYB{Ue8AE)6Xg9Y<%~DG2w~RFy1jCxvN|brn zb9YF#Vl+fdL>2CRUeP=}tn>-zC@B*6o*8*Gkm0G@_T@HZR)Y?Efl0SVc^>#%k^=6~ zEk1^gS_EJUyL1Dgq@+YgM+flfv4(!*22da{@~4Bl1FTj=KeVno4E9UC$A39MxrRj3 zY_|}&T`>!p1jOltgu6Ur6L3Yd1Z!wc(MRv6dS8%$;s_u%I3$G2dg>Q^W3hZAw4XrL z#Of*o+&WWgC6t)R`S0SWe5hqvh)?X!{TzbHUoYN zmq=3y-{GB!^LepcH)9Y0Et#etIz`@s=0qGR+mAeQ2xr6Que``(8| zrhI{!1E&Wax3Jt{=3&1!*0Vjga4kYCq~;ULB}@9tSM^kL+<7_Zi=oF>63bE)$k&RM zzRjZZI_@efYZVFXaIXr}$`rAnNr@%H_kTZWY|*w6cC}^ozm?nD|2;~4e}ms${Zj{T z)a7Mc#Xd$u&6fe91N{@TJB$z+6so61-h3ia^IZ9WG*<3o7yzn-Q6lgt0yVa_w$|6z zCpdr+s&_YVpaHRaco@?Gqg>`-AqWWy0;B;=)Ujecq8q~);*nF}AHJGg3S5PWArSsB zFfr-hR3!lJ1o2NU>6ro*eY7AhcBF}^!DxB$ZyCX55+>29}$3gw1gA9HLqZ{A|WAfXhmB`$7ZT#M_(^R z4vZZz3j~C#!rRlhbRC*wSKZYl`lL?sq?hPvTi=W4=*C!1pFZYDMtn2HW0o*esKxO+ z?5Sc@Oz5McSR*4@3ArusA4V!D`ewL1{wYzK`}S>$bmo(R9G8T#H=3%95#nYMAF0IY zMeLZ`uSqEQ2{Ql@!VfS@NEiQ1uh{z0Ea8u9=-jhNBq7@@k_yvziTnPuYrH_b-#%q= z=j!>)s&hPmKT$yS4rMB zhKiUyvBC80xrV&ySUP{Akjyt3M^LAv^Bte^H1&k}mS+cQoMVdT`f-qDeeB7;b4i{u z)}4N&(B`Cws**vbBZpj3aDvE%Z24}Aj^(Egms^I4Ez3GDrFsUU^D_vi?gulFHR*Iq z^bW)nMu}k%egLEK^XEP#(7$SOSQ+$S{Mh!lBZ}^(+;8wlfIdb^4FH{V^-9!xAOi=pfgydHe=XK(H^Ffed(R+Y#ws1-3$lcM`T zsshJxExW9=xU>McXzfUPzJWh`_PvSBB%0Oib%a{tHvF;U!+l=fKwLrosIqoEk335? zcM&8Bqf8s>8}w-z`)2$PBb4PB?y^x;Mk*~>yl9u4R}4&h`&ooq#4a=R4oW{?pAvxx z9-2rbfs6t{KlV!n0=%zfOnv27;{-nFzQ5_Q#Ia#VUs#x5woLv=qDW0YR_CMJTL`M| zxYa7inX6$Sh%Gt4H<9Sl3XR3j~5qSWS$v=Ebmu#;MivOq*wK#= zGLeEQuq8H_%Y#SkK>zL@KsB8Pmw~A%ZI}4y=!Jy^6+j5ELno)F&_k|*NeM7C)6&uc zg6ke5Tkd-Q{yhhn44ysv1$HtXo{7oHQ$}jqEjaLyfR!1dKYS2_nME+qciLKB8TkQ2 z9B$2C0&4|J0hSGn;zASnZmA0JgsJK2_X}{SZ$2!Ml!u5bEVR;)lfIfFXws9q`=nU6 zws&u72W=be1s(w=f+kYRhmM9yE@F?C|F_JJy_QsnIzqyT%Kxq{siL$5PL?R`i72~V zesp+r;R6T1xZ7-0nBmy%cd#Yao+aF;p^EsJPM$IdiQDfC#jbSPM0CRzoS_RA$(Ssu0LYRiW zaQi*5w92(~?K??(M5qnAasZPK--ZOzFN&7r(Pn4UYaRN@kxcg=Nv40V204weN;ltG z>`|d+f$rm^zas1J*bnz4U(HlqP6kSQpD{%}9`DMmYqmUnh*wR|o;7yVmpRhUDHJE< zeAIm&MmIicM4OdgTV`8v&4#jP0^4GjGi2gCs+v{M?S^^r?eg)h`FVVywxy*dwR$J3 z2Yi24SM!wba_~<=I~r0U0~1*HbF#7)w9-JVx;ne;=8C#8n|^+NpooLb?fBn6n0m5; z(hZ6z>@|%#8g%%LJ3m1|Z3deM1}^PY*Mv zF!;Nxv-hvtzn^QU(Q<2{OVyTK3=a(j?S`syyU;*|H;hmMKmf;sQ#l;>4GQO2jYSHKIxAV?mzO^$B6v=C~yf> z*}!a!$omq$OD-`BE-5*W5~$%*dME?F_g$`1H7$!3Hv^?`C3^4{)Bd)X+lhr@#9IuN zsqr8Q5Eat;{H&v;b}tc?k@5Tx76JyI?cbGA7i!e9%@u5(NB5-1mY=3`pdxjQTCxrQ zmMSh)&QmC4aBfa&^)}8$*ra}De0IxPH7&gTcd*9}ICeLnx$ zamp0s^uDpcNnR(u>*n=(fvJOQr}fEsQ;w^6e@vI=qj(Jj6uaV=#Kv-`+}d8s#6Q#g zCQ~Q6HS3N^ug@xr?w><=FUqwfuTkWk!V+t9)KxZ5Oi9rNblTNxeGfpI}DY9^2_RZtaK#}iciRyWuKvpg8uuB$y5Xp zCtvKb=X^YVc`eg5M{ff2ZB}AF?GudHBjSO&@24oK%n!+_k*~T3xY!fq#`CP*Mcb*n z4z=c6w7fAST{qX0TWG|{YgE$D&AyPL#z^PkBA>B!Zf`U1I6P1}o_{S&g&^Bg?MrHX zAviOHxVyJ>DdkoU_}MJh>N67skmr9x+dk+KDje6Dx)ZHS=S@!@qbrbbYrd7KQw+Oa zz7E1^@cG`gO`?JG$EOn~j-@w}Z%mSSJ-ONg*p?5|L%7QxgUObwMZ zfP?I6uDvFqsmTlc2B1=s5EB#Q;gvvN1lq=tG!y(ub2Sh`ljGx8D88p>1K1m&F8Yl+ zm`*9yC|?A2C_*&=)z()~jSL%Rx{lVJ;19 zd}#dqjWAB=X^{^eaA1AlyYYnZ@XpNJiN{GRwyGGj8KOO9h?b=Axi60Vc1=O9iThGZ zT*E*j=UjY$qp({jN_1tBhbPH05=nJqGm@<(KCOtGbIB3~nUYGB_D)b6_CtIr5`3Ab z;-7ee$hGG*MpGRl8J@${7kH)j`F~x*j|F0X4xNvzpZM&L7=C#KYw9 zQ!2-k95`NhUU*x%M$v!st{_1DS~y;`$DBj7n;5^i;FT(1Z+-I#>t9_X@A>M=sl>^{ zbJUaM2J-ICLw-yh!|flBtyNI&AGK~p;nEP0=lo{Ph^akJiaKxjS)Z2KOe}ms>*>qMLz*+Laz}vi`0wApAH4L3$pz(n)s1s=c`S0N zgJQDjfedhhQ88WdN0>>{yF3GB5zNy2lQzvAMrAOPhj!xzDJf~p;R~SgpePjrr2{Wm zTWf1DxCl=hk|{t3=-UHh_C-Sv^ig5r6WlD|IUZh3|4{~){V_hSp{`r=CM~?Wi3O-$ zPof?s0>28w3;Zh(Xx@XhB4T1dEL|WtE^{vnLEl-tRGeakJPwOO(HvvqiLc^CbcXYa5q>B_ zt@eC!3Ir|vHcOpp;s6%(RfYEbsU>_f-=asRn|-Pp6D52ro9uZ)*~FW5>}iw^fErHdjOclN57-6xMe zO*svw-C;|VdfRv)_Mz%w^_2AGQB)1{#$pkDuah8U1@YpeeZfKh5a;9?FXx{td-z9K zDNS^zC!_B$U(L^tkM2hmp9i_<&ubj!w@&%Ch+aO?KNKv+y7}0ayI2=1p@q&*@s2E7Gt^Z zH?WZkg{FIT-59QEz18pPUt~-=YsLO;Fgdk3XroaD)l<tkGAt4pYEI+O@kwa)Qy#{upOab0S z0ZuNxQ(P&6oo4(8%DypArparp!cx+nJ>F3%qGFUPWC>p}=O7hj8nK9GmtYH)roC+* z%~0x^l@$CMH6oZLdQ}ZScW@{~I8)?cmGrty301OcyArv?5VgdL1kI)4R>NEdLmih_ zZ};>xdji%VKD{Egif*7^-u=N(%+wDCy+8F|!;o~R7y2@YmWz@}nn{e7NsJP$D|Cn8 zqu*^wA@SKpwxy%?*P{*WFQV?-Hc?*GxI1r7cdSq29CdK_MRm1=kRMXJ=jm70)Yfa^ zw7lxpmlXE@h-T#cEbbdhM~}t98~fPo@JAAw>$Kwq{yl+6Yy5i*%KIkBWQM zuUQu?+dc|eIM$Ln_`^0}udi(S>f~ZUd|%Au+tbUjMc)q=UYV*mohY|R6)=l>>N)$p z_?&M>?dj^pZ=bGCg{w&Z(cWQVw9T0-^15Mo@1Z$u_raQD53EX06g0}!*!)|$ z5w0RNA7<3^$WF8Ri}(0ZNrmOnf~cge`RbKbH5O5RRxfl~Z`{}uQ!+hIWlZIMG`jX> z_UJ-11ED^vt?Jgdz*0-or3cY3{Q_!!R$4DDIP80tZ1nkaX=6`x@iWftUwwYPl}j5s z88{VT{It@+vf`V?EQR5ZUYYGq|I)A0a>sus?Hl2A@42j|VLelSwP^1TuG!+!@0S*{ zzNWiBTihSbynJQAH_)cAU$M05L!Xsu>lCFZg-;=H$VGea9)rEVm->JAKB-wKsye-*Rve!>~lFlfvR#0iAzI$|E-t4&gMa{DZCKR>K zU8db6E+I(y6_P6^{rRt`xB)szJ6UDu|9}S-=#e*ryP=r{$IbA zQl`#8FODql=8w(-f>w!pa&^m(O;QzQtG;qQMkb4~LS4pRTe|i)g z9L&Wpel z8e_MsX4ifG=-^%56FQd1TAbJ#WHP)uglyM`W|NyEW%S}+99NO%jcYg?WD<8Y?kJnI z=3Y+=aQ|jQsbfm#pP%swI9eh4Lhy7$+R} ze-!37v$#XUXY%ue_QICx)3KjFWq)7l*>d#Yk$(w;JcHY{T(u1^Unu!Eqj$|j^O^sF z!#$y9+kXEV8CP0+Vf1p2OZuFDNLQ*VSFv5C@^sY-bL^Vu&JND{sl-dAseM5_40bpF zo`8GUwvn09`L|fqkDo7Cq@qVA1fa@Y(*>Y2(z|#K*Xb%!iv+Jxb?J)vCLBP3^j|#@hR> z_ri|*C$~f{Jo5;)(%kYk#O1wxprc>9z}mm75+}q|PCG+d=`+i5+?`Yy5ZeS;QcK%xz{n34;za>@LgOf1bVqb1s!@n9pA8=aW zp>xa7&Q1GsH*DMpvSb;`STwVX4VqJsSZ}mgM^N_~hDC`JblY3J%DJ?c&!c-L92X2P zDVaoL3w^>j=z!5twWTb7tMLbsk|ZVu9rzP;IGDS+%zCEgenufpBXH;-@8EsRSnd(; zj8E&rpMzKl^fovF^p@R@tX&){!e4OU&}xAEAE|r&C?D!PSbhA;zH8H- zsskGm)Jko3@T;=(C@PYkSl@30~p8rq&ocFGXK!jmG|Ba}HFeoi4j6r2c3ag#_(bPL+7UpRZ)t<0^apic`yB=vb%ne^K; zHD#|S>;*n{-c8oedV4B!2fizs(T$jx7z)CbH*^?i7agm4W-L>PwgUa>RttJB*s0KL zZMJ9tuLdo@G2p6p;MS^MEvWv0VKUUexvP&qcDnSlqW=R7$KGRPiQ*P)3b`YdZw`h1 zGefPh6V6r&`@NviqIDmQSeutjF zcyS1lFhxaP0uES!hv%;8MiG$!lrQEv#x$c9gIHlimMeu@aon01-Upgy-B&#D=564@ zn^THBYsLBk0b58pWO^fBx*!%hZXuFOzD!)ahr0p?B|L_Xz)GSYYP)WCpYzGxojZlp zo*h1GRU&-MD*d{G^CS79?(Z?8_sAz=yqe@I*Aap}Sam4dwW2*_uUay-jS9OQ>`;#p zBPuqtrp7s`ico@_(%!jK-<-R>H>>r?#P6fkE3$kSwC|)@c|lh`EJXXKqx*XgTY-aa zoCH|NT|^LdgH-M^y2Kmm4ww%8dS8%L9?d897@Z7ZyZzUs8t043uL~OeCAr@J!e44Cq`| z{6ku|-r)CEPBU>BY!+GXRu{O~whj_tc*_w;Qgqzi+D?Xq?O|*@Li8j$0*PCp`9b~D zp9E^l*<)MW!;=o*=TCQ#?~utjyd&J<5rHuB^`XsuE9pUgcN-Q#?))jWb)Jlb<1tDd z>Pp;)=96B%*VA!kb6{g8^28g(IvEu9dXK6e%zEo2@o#u8x~_`(s_jzWB=>{2r=$~jL z4MY}PVZdqWpNpnJzpg`EK==N{_5l#v#o3;7?EEtGN2==pk*l<8EAqixZjo#N|1r^X z->OA4yf3t+`1J+&wZw@QN8G;ruQfEjAJ`6%`9r{)e&@sJ1t83^3;kC?s9i2um!RtU zZU|t^q9eQQo42PFLwUsVn3?4ajf~dm=!6r32rL{aZk`lEc)c1kBZVm}vL$XGX^4qU zN00J^w?gpvj66b}L)pFV;}iT79qXMkyxhyasNuUj?{32p{o=%6I9-dNW4MES^C2V7 zCgpeKQIQH1rk3EKY%-yB$8&WtUL$9>>fJmCR2^aT(MI|9rSy~wJ>S`prOHU(>%)Ee zr%sK=2q}T^J7az-leN%k5Fy~k)@F5kSS=(c3lKhy`$9FGm>5A>!@#lxWjwS;AUqsK z?ci8=WpxuI{-+_ihwdgly$7ij&JOYG8ip{~mOXgyIo>LCgco38cVFRXha`lfjZeFr zG8JaY$5Qbp*}1sn4i&x+EmxaUxUxLYwsCtUV>0ce)SU5Fe!jkph;vtCK65Nj4gaf7 zK4tmuDP4oC*5hflgKJa48{%>!?o9b7XzIUWtA08vB=#DP^Z8+myCwr;zwnv|KP*+6}R>ayzv9A}9v&KEb9t^4O^r*--V{g)D?aHZ?=g)c4CF=W&Mcv>4m?u}4(K@M@@UPJSylv!l@;T6+aIFtK{+_T}q5BGRi4XY) z2kiw-RxdocgC{1~(6G<8jQ!g6>kLGxbp%lc0#T<~ka%zG8Ht{dOXhcK5mel3d|H>0 z!b|mvZF~6XYf5|LgAA{zuO}2_ICPSF}l^FcvDbA5sKXK6$k zyzMbETM#~O z|5;RXpUM5!>|0CJoLgS@kcSB32v73IpVwoe2?(+P)|f}7IKf})ZZPSG#i zNAFN0 zw{_jo>Q1^6pEK4U_Lqe3P1uWS3~462d~jfZU-pO%28u`-Fj<;4ns~G`t?r6hiw!)) z-p|7SY1Mld>B`?10>y06bKm?+XtA!yYCQ`Jba&-l4#ccAjY>FoK%N&K76y}~esJrM zt=fLaZtGG8GP&~2e^PwgwxxxITe~j4J2|y{jwm6)L~#FB{S>OeZ9o8hc)iy3XJ9Ps z+@#RYwIQRs*dE9%u%Q9tttf+)m4Ix)lAZI0*XX4$o zmaD5q&_KX=fWm9x-;cw`yAo!OATYe#8hClsLUtqzrM0$E!EvApM?$O@_wJw^DL?ZW zQ2`~|qUFbC0Eo6Yw*H8%&G^*m>*TD9ol1^vtzP~YE##xOWAv1AV043isCC%o9bksV zC`gI(h~7)2PtsLTJtC_TNz5h_t@AeCv+&}futnozU&5xqrOWM#v?+VSVW4 zjLrNEf`Z|}k$Nlyhy_59uu9+mBDILJW9Avhty>l`3PfH#4Pj2EcRJ48aq>Cd+!`k~ zIG#wDkjR$!l+)zq#q1=(W|6ko4$(lWw+qkHi+|}@H)>SK=FA9R9}X;l*#`(1TNi0J z3V9S(6nms%VF4Oib2SBvnXEeP)qfX-vI!oG3)6I5?f7P(Hk@(VZ#PmjLsQ zQP(oLH=5Mq7=J(czCZ0fE5|h4C$ikCTexWfdwCBWoz{FE*VqL}If)!X#PiK`LFD*E z${z#Qn0IFc8R(@L2}JrqnRW#so`Z%B@jL`#!n?aiMtASp#nEw=KiWaQ={tYu=5T7T z4vP`Zrqrsa{e+F>K+^VE4~d(a*W-lrnIg@N`?*!ZnZy|4!(j4|?(1CVGv|D7_gh&Q z+sbkI`E6*oG*-x&I6ymt(3OGQlLjn>j8C?^b{E}!yQ>~Id&|@r2)l93X;lF-BmfN& zc)_LrKDA{X53FC||5SkQx^89G?!<`!STG@(7H9Ut;*EAQWm|4(_;~urBdMB-$7+WI z`^Gn0jA=UNY!ed`_&?y|-jgg~GcpJps6Z!Vk>7E6#GZmX?-LyyEb@ zaTjdH6c_ZVWB43rE}m@zz%gFYSBn-1(--91mG7#p>E~*HzN#4OshCW7fA&!E5h22> zn>&{Lio>Afheg84(&PV5ykJ8q>G0-`?Db9E9iNm=qZnC`W?Y}D7})S7k?-W(9;OlF zE}q8hCnfrny#&Lc?B&?_pfK*R7B&55R$U)GQAcxLM;1y@Ouf5}OysC@=_i@5&Sioz zLMxbwyMi^c~`z`0&OByBW^w0;8piSvcog;1P8 z9?&6_L-!BdZG_(LPTByJ%!=AyadZfJBobnb zTCYYqUwhcMuks~F=;p5uxyFz4G^ipZR;TSB?KzPm63$O;`8zl+eYbW`S!UK-Ji`V5 z$(p&LK?sw>IfTsNb}6wx4IQ3_l$24)T_yr#cM8Yr2~hSQQjX8W*Es(5sm0+mK9ZQ&)=U4ON{A6uIDuRdNwc)XlW_VyM6(V=H;*5@ zkxkKbeud3hTo)RZ`G)os%mg7yMTq??aD~xYeqk`_`ABr~u~&n?dOW|^bJXzTbXwK5*ZAdje$rKdyUc&3?h&&UEjyaV~ z)EWAE;G|}Z0$GpViG`6Pl{kPVXW0FbV1|(J1AYcW0%7p$LtDW?{-Q%yZxtr0 zhBv5>P@EJv(yvx+{_1AK&9h!2NO0siqx@rbgKN8YDkwxLBs!%j?ztoOBC~~+!-3o^ zF~ z7)2Bni9_cGb|3PH^%5`@WP%z573S^?luDg1U*@5j5z?Y@G2vnM1V;i<6v7^j$=!Kf zTxotd{Ty(Pxv5TV0ofxot7sTK&=U6B#)f?$5FV{!+r2aF|8W6)1E^ctKtSlk%lP!j z5%U+#3xB>x%hj?FP!4l__>S=+JjJdQN6@_Tz$nNv^@kn?A2xQ=&O)7FdNa*6yk}b7 zkiISf>MB^c#_C7-mRJ_tKkbq|_$54SvN-jv(9HJ#o={@#iiA5WJ9EybY+VZDb9IZ| zhbUA-jjLxE^(aUd5&D0m`zm ze7uh@MoqlwM_1iT$}jHqXb|wE*8`bed)@K)@87@wb1eB*R8k9(O$(ZXUo*=B)WpPT z>=0lGl8#x{LK|Cw0!zUDvVyX*H|*wQqiN0_G@}^!ejVV-x_9pz4K?|8=~ro|Ru3#2 z(P$hvZ~$#BD%s(8p^9;|^8NpBvtZ1jjBmSb;sVtim!upA3)@75SC}H@TbF#BnvVtP zV`Z*xZ9O~t(m=G_s`q(R(13cf&bZ@8mg6y%1>%GehbzyW$FIeQv-p&-j`^kgS#sB| zYS_90gIOERx{>Mo4=q~}q9h8K$L-8-+S_O0lM6LI+yu1;CUVfLNkNnvu^gwX@sXVJTiTYGs>Tn*ac+shpVR5tz=QASpy9Qv%!Eu z*=55y3p=ez3@cGcm((s6eksKRqhnxF#_R*Z%dY%PD(p2;U%-D#@q8m^gyT?4Jn$Ig z7-95faH9e-C+>5RHvkA!Z~-&j)_}-tV8;cIrPBJ`ZBrN4R^|ZwYMy#?2cP2W+B+u2%t?C6&+aoa;?bkcYVhdRZ&JR1 z`;;tm_}`O;A#|Fp&J9B)HCENuC0rpP&lcya`@aaC{kEW@aM`5ocNwdl((I4_4vlqX zaoKM??W~?5<-E{lu@EUnjosG|8nO~hPKJqvD7mpklJ#?^`6*kZ{UyUl#I!7I(?x7CR2P;(Z_{=UQbsZEivwybHC)(t0-=+V(xsBkj zwdPx_;MuIr1+H1&(Fq1C_{_Izbxj}TQ(Lb7Q)dbSojAMiTt$yTjNn_e&|o@X#mG$U z2f>f~caBs8r9fy?1Fi`cacIbYzd}Q4)53X<3u0@zabcEOw zI|J;T<=)~D;mYHS3Y}^FrVI_A?M?e{m2N5OrlFkK*<@{P?JjTrAewL!mj+J=2*9QM zXL<9#OFJ!#ZuPoPLUDsKG$5@5ATz+J{>L^n-z%{927dko9@+s4*Ce7TI(JOV4mkIi zwcrxMM--N@rCkOk2eVX{W~R6}rsWQN=Q_7!Q81F+%RcV^tIp+jo%6i3o4(%l;rQQQ z{tmiK)tbbbO;5K4$1vO>cMGhOdGIQnVAB7+Z3AoAfoB|rn<&j{gxmsU;cPO!DA`e1 z*HyRh7*kkWiwxuQI669cQyq7erpDCt7wr_ONLFU`$T*J#a_&BH)vJ`D3?)%U`XKp( z#*B(N@`8jB9To!92uEbFvYOe(SxY4;t(~&63$lj~s>$#Cz~e7I>^^iWTa(rK+N~@_ zj}Lf(*D-u!*C^Ip0DOdB$;U1gpi+jHFidq@KlOOFD~ZCNNKx@Im`SgOYFP( zlOk)mvVh?>A$xoKjsXNz zy1MwQ^jn}}aBdxi zlMpP1^W^@%S#A=&hvC_gx+_O_L*GN2Tj1M$Z!e4zZF19RY+349Kv`M0G*N{Fr!^K9 zX#)@^EKJ7p;8%bUnD>B?}V7g#kcY}HyDD5|);JUAiFBAT6O>bRLe)MKPS zL=h8o;p}pXVbCX$bT%HEP?nM)5Rz10Bpz0{Lu6&*F=~?Mj#JocP9hWpZI3NZ(P@4P zFuy8Ye*dAmtd8&)2E~ok7lKMM;k^^wxjIrp`r$2t48-vI?r4&y4*MX74mFW%vA>It zen)tO!GxyuU~xv(>{6<9g z*u6?8RzA5bWGNbJOm?}G$aA2kj4?(`(OE%eyQVNt4mtdlf{@Y5R+a`KX<4$OQ;={< z!Iiff2`}Q6nhA6~hTBhX5f$g==F#V*SQXK2Z_E%?%df zUV{?zH=jxI1(Y3WDi4x1{Y*b*W=Z7H+|`qlLFZ&nD4`{uj_9{t#`uQ=BZVBrLidhT#BIpNlp)B>r)MZi&^6L# zsTcHH{UWYI5fzS6A{>v532t}GPyUV>p4x_&LZzR-W|OJ(VLHTU2SFxgYD}ep^J7rHPbv^~?nnDT5yH2NBezf~y=-pw9eXde z@*R;mN%ElH*_W`paP1oY`*x-ezZWb4j0X~p68zMJF?T@%m_1jwqH?@G2pA~Mw6)aw zH^KxMNBTE!)m+l6;%h{1s|+VRcU@NYFsG@Z8hk_w(HkFiy>U3enNj z&U~Bq*4NpONPSFVfVpsQIlZ2 zEym_xhF9;i`E*(dYVs#q)CI`gEIjetNhiqS48t5n!>zs+ndHnxD&9TY$*++PP?@k^Skb zWVwa61~q_4@3twBL_X?mC`XnS7cDxyxoo<4!Foga$%=Co@h+W5DHNv`wQWQvlZD~9 zLSMQ#b9%?0wM++HHOV=O`o2j+utJ|rgny&tx{)KsMr1=aac1>(A{~kToU9mb>l=1m zQ*)tB2+|L=?j2p;Ahf|8qXB}0;2n)xK}AMSTZMh*vajekaz+)V!t zTwPiI#?G2*UVgun-Zai;^H!B@?k7JKsnhvr^n?~c0zFAo(dCP9 z!KxI=T>Xcs=u4^AAU<9e9wA3zAu-~~@TNG0@N84bUC-7L8hb9Qg`?rBUOht z*rVBm_%ZsKf29x;gH%$QSeU$sUW#w{bB=I&HAix=l(c)Py79%YfgzdxuIP9Ux$z&!352lfmOu~+XC z?Q1~VaHf^OXbSs(pu58v(1wjACFk&{fcyO4+C9>qYqUPLDFCjcSulB=ZCC9Zg7ZH9 zVxT&g9-5My769Wc&W`^&uLih)rr*bdT6CdpkHZfK*@kjd4_%4RjDr-Gc05F3g&`WC zwy|1A0T~;}x~@=wthNNME!n5H&sxge+7a4HSO5BTmSw>S24>%aqg92c^Na>^D@(;a zI!n`%P~5JWe|aL}y4*K#-o-Nc2{-M)swWNJADbKh=(Sx($Y$UHuP3&l@@& z|G_VxB4r*NB=(FfoHHgLu4ib$rpav}X5Eny_4jYG>@kgivFQb`krU_o`om&$jH?nu zM8)(D=pJ)Dd@Pa}JbJ?%9TFF9QX}wR2NI5Q|Kz+n&%tep+BZ0a)gFR?ikq1+g@FQs z-gf&T*yYlEkpuJ=^j+WQzeVT*)nPL)FV?!T5&qXUMqg3^e+Nn@A$8Ae7~eLV`UC&M zAOI>BD;x&K49D)qJw78V^wY#*f*o-vS8Gp8DAKCloF<>mxF^1~-<>@Q1LV#k-Q z^itwPMIDE<%*-t@3a)Hj4!LT-8|lIwKFMH8_b5F?)TJfmpE;8rxtWfeCA-&K%_KqQ zYuD>v9S?d&Ix{tW6bBj?rrtSJkdmqBY%sCd97jC3aZxQGgqtPf;yA4|0CLI$^yGDY zX%eD4z-fG;+mBQ}Jk~Q33v*QPi}#*IVdjT>kBnoz77UPO>_4cK18v}uhsPVjST}&j z8~EpY10XWN@OTV;fSVSf)vzks>9{Q;E>0fs8SGEc+Cv+O(cVd2(N928Xu2eLYbUAf zKFq}fD|dht9lgCZ7_;86|AfCAcO?hV8G)vNghpn`_!-)NAZWCYHEn$mMjN1fI(KG{ z8^6A!Cu?8(dvdabvFvwPx0}?VLYoTraa^C#FH%thLp3w3JvW;JeC(RF5}c)RRIAr7 zzj1g|V_9HZn9XOLMaP)*H>|~w*Qnsysmg}ncgEHkqJxo<6yA8ALMJazTahA?0O@1u zTFWy%GF6X-T|drP>EH%pTwwyc5H~#wZ_HtnswU}(yg1&g580*G(cMe$ZWdzJW6DrE z!NL(|NKdbrC_xveZyfRS&%b}q(lflp75^Q1p6ZonYOrV3z0`(LaYnQLuqL^IvHR$r zJKn-^an79C)ffLuL2q|SBlk?rnf*h!!DRPsUtb+!&EYp=_gXi>1$i?cpYq99d<2{F zGlvoEPz+=a)zc6h&|7e#ZSQ|V6<+`GqqX>*BDVOpW86|XV7Z}ZjLgRqx(%WwyIghL z%C|Y3Jx$aP;1Y8lH`Fsm13(gqa5opWXFeJx_;Khs#i3U@{ z<+)f>z{e2z@wV?JT8oO#S0(x}ebD8FV;2V3PvXrSt~`I`V+UXArKai?*Wb$w1=}W~ z7%$}77k}Y9Jax{Y|8dEeCv33m^(|B~eodHf`Xw0eWoo`X+R-KRg6#+oEJwgDBo4*U zi3TfL>}?h_q9=%w>7;%z$MM8F*wE>+B^b(g&>awZPjn>GWj`*~B~r;@%7++|Bhstn zML2nR<3&qtWg;W%gW^OTV?4q$#H^DvA{mbx2S>`)B&q@;{P!YyscCL?y6M}mI)lSO zP=H8X)|rAr`8Mn?<+i$)+2S(85I7fk98lq} z>#E=0xS6UrB4fi-_&^WbS3oUV-q!X?xgC^`>V@HSS~GL}_;)1jQk`}*AC|%N>rj}D zW77fnSumY)(%$(}xgF04YoYqV99UcE3A zr^5cOj~`XR-d_nq*^ObKw1$Spim2Jn8ay+Uh_GT;PcKl9v?;WFl(}0I6IYFzv<=$^ znt1ExYSh(s{rJ8+^-tSK0j(@x+Z(9p_lwIvy7~_Y-Nj#C$nTmN^Wz0h(MAqjNbI6$f}i$)J8_5mi6JLVvfyf-1)ql>FHU` zSv+UY23JLF7VDmPk;+{t&rHV}B2p&NXXI&W$DFF%j+%Io}+ z(s0y=qC@Ez^>Go2Ee++MBj?{#xH-GNc%YM34PaQxjx9;^AKQN8Qud>WMh9``@ZGo? z4>e&#cdUeKLOrm*tL?US9Tsz^8I*vdpWWKxN=SI6>ET@P&Z@S$!h7XHAE^% z6X3yg0Du`D5e7n%?8m0DDJxUim70hpkSrMWjplREpNr82O^vg}S)>u^}WQqxFjZCjF3g}J$WdsB)6Bjd?C z4B6Sj##8I|#*TU``&vCD5aYz|yrF#Q84*BpF-1lbG)9DbRH{STXxMcJH@`?yL{tJbF zF$$Z(wwb9Z|Izlk;oF@c5^NL{q=^crH6eLlPe9bcE>qLV!$p5)XIIhOgLB70044Dh z;9+!pw_Asz06g!3nL9Sy!nmRx2o=827j0MG(wz26?AHKZg*)W?wA{RSmcoZ0j)O-h znrH1wV^5=p#c%;3N`!!Ag|+ho(O{EkjqL(=1pCzv^#4HH104(+F05+h(~H5F0VOwh zFJ_t*iHV8e+0fAcr5#JsC486WdY|d)_P(1m;Wmn}5W~8)I=8*sEM955lw&IDg?_3JPJ2)=EFE}mY3qw6ha!*dE&`= zOgW9+UAsd6FRg3<~k(0~kmc-I z!>8u&)YR*c)mqwpnCR{4KcBe6&KDsJP|mC3e!Sy>-So=OmyR9$+j3k}^jw8He8s3T zJF1M0gI}2$_=x8exjDoHmwGcN^>fnE>zms~m6Fol6x<}rQ*vLOj}$Fr^b!?5(%mh8 zvWUZ2P49H*#jyr#mt`D^_< z3pQ-ipU1Fs4X${fZP2Eo)ZYmYR^b1t=d{Qhd*ovPoj$W_?(CEcu$c!=0rP=h zmqvg7dWcyir-G!dToF6X+IwLlns3XQyf}}Vbs~$a%%~_SA?n^j^+gFx7 zmdeMiW!E|gt%;I{cE^%OxDu^SQNmoU-WW1c4717Gy+25*OJC3U`|E!t_v4#bKm(zH z_Sk0tqs+|A_D?-eqVS@@C?-!JBKnT8B&J>{Vvhp(@0zTzkpV%OCdRRObno;Ck{af$ z-hD#X(8FNEdl{|tz&#+;qeeHlzUdGau3%c`{O#%&;5tAdAPP4DWe79^P^?mGj`Ri0 z9mR@xtfpe;lYEF-AM(|+sw!={$+D*Y6nyK(ue!>{gs~?Rp%0%*2BkEXPcP<0oFp|53xsOuEHW|Zl9(IfALd`Eg$>*-xHmvD zzPeYqP1{-+0-Pm%lyIt*pC$~{jQBc?LUS5UpAf`KGR8OlPYFF<) z$(J!IW|B3L4ijZek`OmCZWbC54l*)od1`Tpj+=L#V+f;|eB52WW9nY#qv;5Yyh3#O z$0YZLg#?k=c&*YM&JR|w3tXAkT8@}(+Hhjf$dEh4=s>@nsg!|~D#bK)uljIYDF-VZ zoh2RbJ+FKUN$LrhTL8Sp;#!r`LKH;Gf;8b2h4LIICL6y@zQEYiVF0Zws5t13 zgLZ&&I{>_Zts3{C=fRg)WRVwp+6+99u8&6~K{djnQ2#R8oD~B!B(WQtRg00%@MBwI z761d~BVr;^N?PQ$;juxxpZR%p=k^pDStKed3N!;6s-49@zI7j@{~#U1$?~tI!4J4~ zoy855EbGV_Mpym^wGAgbJNy5(&VSLyZc9+KAMlEmhOb(e^ij@E(BHwHinH2215R#wibnLJx`aYbX-032 z@$5qbl2cmT}%-H^^H9c1y2Y?2RX3YRH9R?s53*B$kHDkC-lOF*}$C_NQ>F+~O za*@LW-4+-=YlTd*Hc z4rARba<{pK#oF)?h?GbZ$txkPbl~Z{scBHhy;N?&H?~%sdoV zx2}poXp^vi^xI5U*=rRRsYCXXs%nZ%I@E05!{9~U9aE#4)AJ7gEDj_Qoj9V%54DA3 zqq2D^aSFto?`qc*)JF>IjB{L`9%Jc}@neY;V_>A03S;QL8@b)?T4eBbx**;9NF75W zkttVsx90^Cv*>dRek@Tt-%21dLENM{tRJ;V++=d!1~NKz%DUo4@R#s2fL3 zZaM5T_6tijPc07KX+zh*4O7HwU2eSk8fmj=hQ3-7u5kKgZTd=zpMJ9qNcnNd6 zHwp+;;Mp>BrK+5@MY2Y;_yiljpddmS%{K-tPTHrUQC?@kwSNut%De~E85TA+^B!<) zs?omF<}H4?E4_Yw7ybRQQ9;Y@%65Yr6FsFa%pdicQ!Cn2T>S*rRGlXdDQP5ShIS|d z3CZ+F09w)9@1Jo^O)kl8P^R7?jFUDZsGr$i^rKjE-s>npe=9(odI1Rjmp2xY)g&I=WytU z0|VFY+##k~u|M#=$79%kKSiEDm4v_#acQV!M`>@lf<-c9b|hgJw2L^+sr7C z3l-Zu8bHWJljZX;FsobXWjyve*7bks*rzdg)6;~G1XiChf{s~4ZRJ%&s(ZG+s*FQ z9oyNtbFjH1O(DR;)40&{QT)5l8^yy2lspkd$KuC)c~%W^qGD9nACi%-LPLd0(U#)t zxsW#J7rymYZLSc;n`jz>!R{REi?+#cu%0?j?viFhB{*U&z|E4+6Lx|u`Vh;FU5gWfE2t!jK zwWtfxaSTnDtD%DMr#AV|tp?6^?CQu}TpOI6Tly+wXMFSVlgMnAq}`+>%2i2m$5VPY zCB;J}?CkGR>hAr1-9`+Ee`Ra@IH%F(@nx~44w;zLr*oN|ODju*UL8+AQYDqTr!Q)4 z`PuPwB-6j+saM6v>Tf01)|3B$Wi^#t8oaKW31DpVx$3_N^cacyp&#)&IfrroAUrom z-`r&&!6Ff!;h$+Sd!lP6Aj!?{l@2I%G;)EIX?+&yN>Cal4B~nKz1u)oz7hyjdb9GW z`Zip$S?6C@*mEr5yGqGMdL3C?rvmJ=Y8kz+!Ja+4(fEMfTlb5;iU^NTP}u`8p7r`= z@h{VVJNQk^U)J)&62{-Rx|Z7S{`&!jz#Og1%p#GU)zzC;pDLeuCq)fyG);MQBBv-z zhV&>>hW@18sk@w2Z=b~$ld*N(`E7p#SlnAol+VN2H zoA^EN(Z>l@CPOonpY$^PAI87KtLgiP9D_SatTH5%J2rPax3!VR3NJ6POrKBeB>mZA zbCb&~hkWkyve&!C>*=az_xxRu+4^|>wS$wP$LuMpF03qcArI~9a`fYU1~nvk^yqyx z9@e`)J(2g|NQcU-AO}-u!||3Y#TULt^`F~=?+LbdZ(x&58UqmnY=4?9F<}&-Gnhkx zbLp2+bR!X7fDbo8ZJWSTz-6d(gdpz_xZ}SLFrx#OB*8on^63p*x7z2Lr2f`Ai2e02 zZa!RoMu^~zrW7N+wL9C3`d%oR`C$0`<#fsQ>(_fhoWX^A86)H+6f}b#m$45rFBM`~ zrz)lw6k5kLSC^!yBK2_mi3Y>klXJSwcOB;q6cDeVA=l`AIguJfe764EmyjgI(H?c zq+EE>oq?_H$g#T@N3$Tnq0PO}_;~5GFQ5>P2K^cIKxioCj#baE%=LBd2Q}u1U$4)i zo~^AdPJ3>=M-_Y|Y#=@|a5=}+Vmmf|uQOxkIqK7W2#FE;%!3#`ks~vRhhxxRedueWF85?=f0*+ReCMh)Wwwhn_(owF zi!)}lUta0`u^L8U;fOrdqcTroJ*X2IDPw+x&c4O{o|5~M(|ak>N_Xi!`H5t17axz@ z6pG81C7IhrlSJNYL06v>i_HuY&9!cKK4|l-R_EvKDL0oH*g(0xNBBvZy@aA7NjgC& zg=ddKX@?pA>j*0{eaJ?KJ@c380*8ZFmQ?SX**?AOL)<+YYRJ!Xfb@#Q7U%w+o;#1R zS42vGGUEJB1B%RR^R~!mefu)fWVveyJUVec&9M#tI~~!@`B+I1xH@(F(_IUjjC})Sz0o&(=1rn5+#Qg!$OA5KU8c^5n^lI6I6u17(IJMMe8x z2-vp;9J1Q7!*O)lnJ%Lfu*n#m!YckZ3?;ej% z7d#n9&-HZvSBZceSH?jyBdJlBFJ1SsP2UmkfFv!OH*9=_%4e3N_HUSr^!}<7O>ZR^ z{UnNR*iKOLJo(=PXow4_hA#kAOt))=H*YT5_E0uNk_G0RBgQ+K3Aof7RtEuSK~(G+Rk8cZa^ouFIR9T4|v9TQw+)Nun79uzUek56ERt?5J?sw~W{>046<-wZ4)-)b( zjy1u|K!^I@W;Z2Y$lR++d$QM2n(1Qz1Eo=hWEIvaHSFvs!4`Il--~hEA!b7}=esKJ zBh9a+>dslQYTDkgJMp9W7qRlH6^rDZyyOf;PwqFi+&rvQ?(_02q{!oRq+cmcPK?s$ zw%AG0GxGBu=O;cn#4VP8yrD>E==s}0qs>&UmC2FxgmUV9q^=Devu=pY{UeU?$qq52 ze7V2QUOMzc@@}xAhsEutom&D(Gu0dEu4!%F`&Qxq5`c=#YuFmXvye!bUs=WFjOKd! zyRU95fhM^LsG0_q6jl}5kXxcxMgkC2ayH@QOW@k1Jldf-Nn1~gHVQQ|P1pG~?MEe; zDrlK7r8)@$3{dB?xyq%L)qPJdV$kpNb|V4XQ8SftX*v+JNz=bRPICRWnUIxW4qBQl z;n}J^@aW|h$;1B^-w3h){(bTCXtqVE*{T=YgP1#x^LhV1SIceIes|?{rzYK}TfBOt zecLu@_L_v*J~&W*;zex>3+ch@NCD>Gma$@zIZ83DE(?^`p9UBiHpKhNkBv$5F!4Bx z3iy(@JTfd{dPWsKS`-)Av45`wA9oKYj7e-F8}DltB<``IggxF!A54`0)E3fspN+4@ zMRdy-uj4ON``T~KeE!?idWSpp>e>D~LFPdm9fnDBz7coxo_M|Q&{Y>X_HD+5|LVUR zbc9cb?c<)0*W$$?%i;oupUH(r5_R*3WLT?#n()W&G)C~3KrK=IdS2he#KiY^R3vTq ziXE!x{=tq|KmiPLD77~taNv0qGM(4=`rybEM}^&i8%ghw^AKFWdcfSq7ZMr#n z*F^vPg zZ?&*UZ)_;gOzw)(LQ`jl%$N828$8t*^KbfiW{eA>E3%% zI}EK}NjwZH^I|6Ggy^#CYI@r+Mlup2WnM=Z6AU9Mgv*kVq(Yt5%gl2VO9NeUR&r~{ zPSPoam=nt*chqn9KEb`amFZ1)HsitiN8>?WbUN(jm-4T1mu)qeeicN|a^hl=HA`e} z`Pcu8%S*03<>QcY7szp;3&A*D;K&YH+1OO{{pli8R`3isg%dC_iHQVK=YjqdzvcA= z!wtL$kl@ggT<$qr0nv6jM&eJiQkE-pX!aSLn^jEaUeSuinfDfpg*3T18duFV34G)J z@X%>WhUBkNb03~d7|mXJ7wXI`5X)v!`9SF5Ws|7@5EHAmb$I z%T$Q_&cts!87X@kS@Mps1#znhQuK&qJqbR&x0g7%*SO0ei%(Wv}v0YAy;$19tN3jlNuuyGC-((=1GPF^fpys>CbX~dywFr!)rhn$$ zrm5>ufzZBxU$b3Ebi1eD=mV2wXQW3p+5 ziKNUzl0q`mpio4rcYW^X_kEw^xc_*{jeB3$bq?!XYn@1~?U=XPd7-YgfGKx{_s|Cn z4qM-+17^eG==q@J>wl}ty@wzt(uQsSQykBeXotgvu^MVet(WVy7M|Jp=iV7_`M&LN zU;k*znI-%8({1OGw7&Z2`qG2^RSSyIqyC~s;W*Hqq`Pvez=oacSkc0*P+>eSIq zlWp0fU0nLPif0T-0RxGrt^6L{H;`BH{J)&(&ns=|3O-47iaRR&M}vU}p(RCt2x{N* z7AT1uV)WZXr@z8J#|G$!?CDx;$N?~dYg!?94Dbr7%b!c7e;yBme*GF8U(D)|5JK@C z3D2sx+A~0AGLbJ~fb?VZZ}(0GeEeHNq5)}P(*79(c#qEPvu}eiQk;jelviJ|sESG% zE&_0w$^l-8xLwa&s`}ZGB4=BONCn+@^-Q%zc_53Mx$4iM2g=r7>@zKg8;>ywKOag8 z^gQi3xO8loykvJO;>Glsp?5?JoxMS@qT5S2usiIk*Rk$B4PG7bVvE$XBo$e9E)pfl zc3OV}S4E6!_7DDn?~WuUqc$!ss$RD}Z0L=HKe0!f#59rJIN9Y$Sq4nmym8T3+a^l3 zb{0)yj&Lp`J37bcaAcU+goe>%o3@tiuZw-g!om{N{W| zBm{w3)-3a7_X34EnZXu)^MKV#vJBlV32)!#O5V4d=Zy7AbJlGvpzBAajORAfxGA5U z{f>y_+`qD|;NF$)uiz?w7XQ*P>t2TheaTRicA3&2dzOzin-ohTa~FiiHt{gX@#h$b zb5vOBNRgSs**eKb_?$!|OP@)QxJ&k5U@OROH!WdGis55dJ*zgtx&H#C>b2W}KE28J zoqB#bsiK-JMrqBZSBlq5Fx630ba=a;P<2ifB>rEuZnRcAr-=Veyo{CK;SJ}B8ZT`?anO0j2k5IFbJor zehOXz3^w%ihRDh6h1J`5*EneZ>={t#2q_aicFgOBo}wT=axQy~pEouy)mBHCo;Hn< zvuEgNQ|GmnzU|O1*IF9FuWF??CJ}YBSnN9``_`}Q?7n4dxd>xYcz8})j76fc%>mmf zdd`V;of3w0dMkHcZOer=Q94zp>$PdyRGXZw`&mV9wut+CJ1cJ7@qc94!q1n~=(g&Q zDYHCDgtV%(FRS@@nUpZEc%8G_>ZuJK;a4dPmv1><@P9zhzI9{Ap$(ma9i(vm5vJ{8 zXGfpSUcR8$A2cL6kQ4O3;)i8N3?|Ix>Bu zA@48uEK#b0KuE}_?jJ|_Ox%NNbu&JA%qpJUqJeZ%d$^HP^0=vSAfz#gx zRotE~Ht?)De^0HJY2iuW%GV1XS;xP7x4%#@HAG`7zrXl<|0joGps0`67>@pJ8m&)m<_CTuEY| zr*4V|Ha^VQqoBvtd&ZJp_*FAX-#DUfs%;w=nU%_|!jn_nr@KDwjIp*juj`)JC|z4| z!D$a6j_@Icvu%zvt1I<l+~mrle=Wm)We6wQjeZ7WhFr z)=1<`0P~`QK45ukRH9g8g4YoMGQlP7W%t0Wa@3J~3-+44^{WM$ikN4jdh#EC&)# z5gsM}k%fh4x>I;|K$Db{R=%t8L{*F;nN0tD+0X3>yN-=61Kl>(mfCc8o1`m)9=|Tl z$?2eI9J8@2v%$M;rYpvZX-T2(lL}v6B|4_`g)%VL(PPt3Na}9e_J3S}U3_HH<9ai9 zkrO;>pH3+{cDY#BcSu=N^1kkrwUeVI+-R%;h1}5dI<%x1=7-N*iw-QaiFIg@BFK}1WX6|)s{SJvfSHGqox8>tH z6RJty{ne{<#7pH!yiUC3B!(_G+cxu&c$MhrVBQPSf?VlWt^KK8rE&^e`!4eFlPp@= zv$}TdfB#}feO;`l<6#Dmf1GiOqt%Hj%Vs~tK%p+uR9ay{Gud^Y}fS@t5t_T#ig9=yq@puSv8wEJ#)Wf>u`2Twa|Qm z_MiX1O%6wv)S7MJfd~V9+vC8!5Tt;jj}vqz$OOU$i2Xk`i0uT2Cd@%QO96mP!M!4K zcw1W=x@t$oDf@d@mKpmoMN&YOyBP>Iri~ICH=g*?r}X%rj{t>f1FVJdr8=}uBp?b%XV2#^ky9uq083cRjYn)7DZ z3=S6(%(hppUN?4br=3jO3f>#;l9+%%^7dbjUl%{bJ;tva%PT!5zp~ZEvzA9}ySCQ9 zw;!K|HM3sv58T8?J?_MCq4)X<0jwHd<>X22~8kA}T>yFsz?5pI~A2j}TD zx`FDjqc}2zr5gI(ZD1Gvl;hj}=ST8}mue8|Ai0BRamqF*2~h|oEHe2N_%!Ge@LXMIAAT1MGLH6ncnZcj#qWVV##8`&`>gALaRKYj5hvk8}LXZsIvd*x8xzCXfN-hA< z0O@^-YXN3IxpEOM6QVLQM=>z~-~E`gvop9dWxwr|=0T`d^yn7n^1uR6u;XFxks=!= z%gw`sANg_dH`WxWBCF{4OkA69#u(Frvv{d`NGi@G2#9xyH=f8^9`OXpisuQsK(>1I@pfMVQfi>utt#AR|DGa?LU_eHoFM8%Ka87P;PR|2Sy8pQks{Zk z*IjUtb0`0zmgRs#+{XkvC6!ZKY~Hs;U2LLzmQOxwUuK|vz120{T6A$D*C|xUKI&JkNsnto2|}JsmvSA4ymmD%Z4{{JuMADc`dffe8^Z$ zoXpBbaY;$<{)*M4O}|#FVdw*<7iv04HD_EdX$2nsR*eI}P2Od0&JqsP5GwP(+&`0$ z9Q2Kh9)P@tz7P`|basvYANA4txz>duo8`ON;ZLwPtoQBcOM^-k==FAde?^2U(TwZ znCTs|Gr#-8NqW@!$bAh*k)^j!%>>UUh@MYVKGkv}I@YtVcu(4nJ8Mb&P6~~0-_UfT zjN>@JO#D^<%gfI~<(C+giPoWu*V9uDi;9LP9-d~Ar4L$;7k(CRQsm*fc_=w!v-6Fc zk>N4gS-UlCJLNmrO23-Pl$`7kdZ9OUWKs3C7Guar{Rk_@hM*K>5pTIp3Dk+mByQu+}%UnERoZfZ-ITTG_yHy=#8<5;#bebT~0ZRkV|S@}LoiqBU87>;j)UH7zZI zf@@>f&Xq&}EYO<`?X$7pg84df`=QutVsy|%Vf0~(sDm@C!2Axc=W=o~vKP02fFvDd z@&WJ@H(C3Wh?RpNpCKnT;vZ9nIB$lc${L z&Tj~o3c9B@-ZD7wz#-LZ@O{r+|4Wa2JYIfMo8I}i{b&Lbyjp{bd}he-l0f(C*H}_X zwU~U8eLXWWrr!LjD&2!UVRTng*}2I5Y8!ZO94)jL=kHtIC8tck98)fHbN_UJE1689 z>xnA}8R;ZaN%F0eZR@MH{oqih7`uy#KK8pHHLKpW@k*fBvN`j9%9TE2y3Ew>-8;2T zD*hQWxtjj$nH=?cQZy6EggRIy0Rp&9w(pV5fPsq#C5nb}P4cuqPs=$ORdg-CWj^Qo zeD2covGoDU0eai3+4l7{rSI|QP>Zwt1Y$IlE7iDxU{O7VWg&rP+q7Phb}|1i?f8X* z;E?f8xL;Q6-&peQ`r7S_D02aR5Cp;6e+Tt3@tK#r7Y*KsVSj4u_z;+E8z4x(l^E=rC&DaEZo*8*G(j(DC$O=t|m3iK5i!T;-*c*`t`muoq6a$yh5rWGug>jr31Y zLV}lti!Ab);@Vp?{XtsK$kop>LZ3HhbOEoEfpFNz=mvjL4T{?1nQGaM8(+kkRqLlMf%}D>v8{vqtlv5~ zl<&JIadG|I$A1<&MV2~zCLX09`*;26)akIxr5=7Uhr&PMH{2sj|taWGANv67Q z*lTp;KyaeW<{i2Ft|zueHFc|)d4-nj*JkG}?scc`!BRWw3*QpQri)}|+x5EC`7%;eqL%5P0T3CUS9AW_%d<(*kJb>l7f^U0a zVIgLTPtly{_h8CZS*dD&EAu`!z0Y+SiD*Hyg;E?~XNHQBmRh{!QNT#Zg@BH{!Ig%! z0>1r;*SogesGWel3$%IFXMP9%ACCppqL=JmJX}~~LCE>A&LXY}Ttuo8q+LL}Q~T^D zO7MH3Ob2{4Ig{r$SiLXnmkF9D0VSWRszK(1t{= z9}M9JypC@1lGN|A{S{4Hu2jBf;*vhhNFqxe<`E?;mnFys9?SkwnVh+;dz-kPt7QB+ z+tWJ(=%vm~_8niXw_V^@YTz@n&>^u>PB?R_Na^U2$&aWvCUq!%Q!wvVg*8KSpXR<*EOHD!;wD^VrPZ)$uuh?1k% zRSyd5|IR44FdTXBt*Nob$GzK>#FW;_l}9^%CpMZjk&@2LmHv(=t?gU!aBMPWgFvVu ztyci$ivDw-K`ifd*RAtv_W#j^w3Ss+<$Mxgt@-14cG#AGZOS_H>n0qP12}9l@$(*eb$zWnD`t8(hLV+h;!E=?{oskl>G8n(>hD!Y zdC!W)ZoO4ptdg*i5*|UTH@&{I{B$GFvAW5)&T|6drB}K0W?40}ILqc*^2@uqhT}*i zx_I$g3rZbX*O9+x-?(1&(gnG!x1@w9Fe}J!;c=~~q0niGneod?vz#JoC{kBbR&cS|vd0LUp2XHF@ z(t^D53cDVF6ek3E5fCuZ5M+q@l899fM1~zlk!3I5H365%>)XS$Nici%!ss>i-UJ97 zn)7Wc(9)t=xNEnx0G0cR>AB-r;z2tpKHKB=-0EfftNlj~{rKdPY8}DSq{X#Z%(74nXcbS(LTRGpRCdmmW3wb@^lH{``CF!iyIew5;g&bVc=xR&K zPGiUrX)dMeFvU>PY&}HSSgx*iKFH5ST9RYfW6G5%CE0n-R`gTdy2dvyqB+d*a&?rg ztCYX0-^tbWI)hHy*`K_8*(VQ=>3Nx!Z5{2j73?H= zomzTVuqKgo?u5-B>*mVW>(9weMeRK1A${C_8)MV7?4ql(j)ow`}=o>Gh`c@J|0vM27W|R`kcmG})G3hTI7D04Q6dmLIOq0JkDm4=%iO3mWh} z|F?fX-V}kBqr3#U;699z4ldzZ$n7h*@-zg#VCc>lMrN)9kAGl?<-MaX-vu=RYB~+t z6H?qTH=1Ev(pht2pE(L^4Ci)89U zZ-VpytQ0GUu0GD{!h9?kbm5$wYkac&Y<$bwDjvt*w=pz-`^P-VMw58lyV}O-tpRkMw-HM2pZ^9y8vcr(fh6MK7Ez z%cyfg!B6 zX>1yjyMoCYV}I$cUS`cz!gO^1CrrewX9|!ri6~_#%{u_9b+AT zug~*K+}zydrTi2oOgjCaezzg~b)YgtFO~ZHylm~zKf<;PWB_=sG`)B=e$cB*bZf=y zI0eHu_XL)O(h4mnGY-vpeHs&-c1TOr5SvNzBruHyf ze+kg@u!&3RHpsi zRg0@sUSGYj(X~IfPg2TJ>1O}!+g2IPZ@kK{iFtKZrkvC(;K=y=P;~Luxfs<*>zg7w zp6};Au=c4StHCEb_s?tjOGcM7!{##MAfi zCn^SjEG{8&M!hmU;j-m7DK_hgJn20T$eFzH0+mgHs8 z?c1ZP^TRRq@3Mek(s`*EaSBPUZ6sGjj5~+a-y$KY^^%)uF3BKfD>cz--Np#>k&73b z>SKR+QAkqE91T*s{b3Vh&#a6VXU5yCwPixN2yh!%W_~gI0ZALmUSI8%1M^wj|==2bd zb2}Zub*VnH0y$=wpA`W7Et9VaOgq)s)pZQ2FWB`H)hV8JYB)g3SJ%_EA(Bs2}qA{0V5jQ*Lk}Y zY>Ec>wzr|zx`RN02bB*?a{#IYRrPlR3;8UbG>Sk92|k{L`NV*|S?1--rH){CN?8~7 zG25u3ivz0zBLrn=r-n-V8X6j)_;(OUz68f}{5L_Jh2A&}C)?fKz3I`@{qC^>Km4xE&HDQv>g5&`T315NmoIxO*KT{@hv}=Um{s1H66dcASFTtY z#0mrkr%>d3xVFY-IEkDsR&=D(i_1&Hvrn+;>z7*G>_5Xyil{S>;N&*R?`PA$(ko#R zPSLu@o)|bbwMzr$J#0zk(?u7yk zDKr)1&M+DIrOva3DsD8{ovJ%+$@%0ME>mOMsmjCwCE0CXY(=E2>iMI#r0zJ$HfOuc zURl7A`Nk-8VR>X)y=ZaO!@$v$o42k<#UFA>edKG?E~=;NcfIGt(@Q{IYF`l=(Y7=Q z+iHCSwF>W8?j0M=n}>7NPM( zlZ>5gd{_|^w+^u#m`193UvpcV5Wrf9R0Fj5Nd&(MaWfyIkgHnN=%~=Cf%y{~Gcqy) zi~lUf4e1!k62x#2r%>V(8_hU50{@JoAb(NeNm2z+wbarIRS1lj5aDal0mtn>ULOaU z8gy;+9C$cDukoRBz=)8XVlef4kgGmS{o&EUH#sk!UtBk6Q|!p#-Q(!yIP_NV9{29B zPz&KL54`_7)HA1RYyW*O(PVz$WNjW*Ord+ko#)@LR~>op(Eh2YQPn`9t{Z!D{ZIdF zeWsLtw$Vk=SzGd79mAFjE~_}-PfN#o7JF7iu$k!7c@t@>fuhu| z*Zamcs2x)=KiQ)67L^WK824-Xp& z0L2%5Z#rP%>L}d(#lT`cN3HIWBNM2L_;IW#f9K^J!_EQlUU>9Z`N)5baTc*>0dpon zK!}jI++3}SS}gLN_9U^)`0Pi#fP zpb1dp8FVxbvnKZtE)ZRh01QO1H!(K;?i^IM<_JMDM1qi|<9Yqx*$RIj@c@F=p>1?_@9lrJaPCCB#%93K3<>C^XR$K84PwkA!Zf$;X zLtW3tfLmOTfhI0Gw-gd{K%PxmMaq!Hsif(Aj#aYs$2BoyCwN!ul3GP5IR@eJ++%Gj zl8fK8s!wlYBGW@zUgV~&Wiqz-9jlm|<8Bb8pGQBP-bhxTST!gpspZEUBgg&sFZEiH zpG(~H=a-)CN@uYt6zxpva5^)&teo>WM{=g}5et>8we&}`*y5oy239lgs^O4`XCDC| z!A$P^8yg3>%}Hp?-AUgU<=&Rx5(I-wN=ri{n00%u1IAc?=@JNi3TV9g)FZ^dVh z+rC_*(!G6%e}I{VsHrY(8xpEL^SP)gWGUdOf`>s}jkWaQBn5iCQFYP5{RhpcUF5h< zI^B^;pG&`(L}f~2-n0MMmh9Ui56-ZrUk&9sG4HG&pc%h@!N^&LhgO)Z~`j?eUVdb(IP^&+^g+<7UlZ%8IR=ZZrdP zJ*6qP>{{1rW1;b=XU$|X)lf|%HSOxXvh38r(He(y?K`T>#hIv%ZtL@ZJY%CpRJt{r zBrWs^O*-suW*|Af80)#M5j44UvDssG&P=QD+n{6T7wohW6?H;Y0+qG*!7)tSQSKZu zuSA$06UTu8()aXI&jA|-`y)Of(84068Z#}Z*q{-B_Nf%^S%k^jzdM11Y-g%-7}yoK zM1=DYe0;S4GGdj1FMh(L!1}`?um|>jTSPQBL^5@%kCM!U{&waE^pn%5qb&;nRv}}* zI-8bR%aY{DfGEgY`a=XYh3-mA^*LF&Q6GhiS%`TK|FusU%OWPI4T;Hy^{ z)58y0S{7~Zzoh-1re0ljIQpDgXxsZIv@^CN)2{^EgQ&#OUi_BwPt|Eaa;kMShGL$~ z{VOCzV@!qDa9f_wopFVQ}zH2KGPYvO)A$|*kzDG@Bsx3o}Sc(9LZ;(2h2v62W zywhtEuU%Wl;0(g&4P5{Bf84n#8*=ewJsK4vyH+%@=*fw_JcnEzLFhnZg+{sMpdVT# zB2D7}pJ33T@cF}^&@kYx7+5TMtG>A^n+m58(cSGPR(Yw1L$=GF8Gc`Qabmm`%&6mI z+jqS59GZ(85@|X}<7Oz=uP+Q(m!SB$$(b%2B4Jz`8M`PFA=d6L!X8HvyPUbb`H_L88W1w$m$p1XxTKay7kW~%v{Dn}Ob8uXdIW8;dF6MeOYamJA^henP5qB?Tb zAbQ2{V+I2+tN*K$pEcMYMoNxV`sO6mG|T#K&M152W8IsuZIU@cPAY~ZSLpqzVJ(+Z z8Ba|c8!0AyP~yb%+7w4cV%5&CU%xPz(F_uhCFoP4sY-ZP_!v+5KO^!cWZOSquJ?|5 z;16Cn`WhIs9Y;5+A28p(?d@Hw$wviuflx2wW!eW%&1LagW7(QIraJ^K1$rT#k(%Nx zfiNJ5)((ZW%0EJjG@%3bhZvpkRmSro4BLQi5_T;sOAq_K9^Z%l1tO%*NyyzLfOgbs zDDqL5$Y;YBUC{2Lvkt1d^d5H%GpDWIA2L2?fxdS7r`F=$FFn)gaJkIufA!ZP#kJ5( z^N+1ema>Awe-)F$;>pipj~Do!PRKmEX6iOH)$2o|9vV+QE#D+D+CFTJ*!~z}niQ>&x1Wuk z#+(!*_c6Pjjl{~tTAqJauWc=d40C-`6T|rKSE@JsPA<-92A=a=WuPNhx<`p5RT6GD z8prRDSmU9@{^Xq`7j@Xb`sE4DOWM+HLTpW3EL7o-kxq=ZA3UF}-rifWZt^VEFo=8g zz?>Z2uR@PTZ=!Ckow#6@|3^FFDGMokKA=vxLOG|S_DWXh@?A8Zm}zg$t0&r@e{!IO z&tY)KFJ+O9sU0XEuW!j>E*d}DAhg$pmh}LhHsI5&40<9b7o&QGhWA~~Jjx+F zz(G)XzkxD<7?_pS=l7z-^D$m{^*xP7K#rMclZYpWa&|7o4v`b9V@=Q#*W2u90QNT2Nrj-BnxwZq@s9t~PoJ3X%+tbW*hecKvRaZ`8v zKL6LLA?Z~uPXhl4Hf4~Xzerb~201xk_fC!bpVMP9-b;I?b`?b3%5_iINo%6F&NT>f z{5*bZlk%daONcn-Bj?7d`c*1i7q0U%l3FUxq>S6z2b@upmC*C!@vrk}B7-JZOsVD*;b!J%WJ}C^T}qPEXwswP>MupQS(ps}9;4-wC$^UnI)d(ru0)cm(ECGy!vY@NXuC7#iMxSZHwLGW$hxJQ(? zWk+^#=BwcZ<870%o3ZEa7QUc$3sYVkL!w115Yo=`!p(fqChXweTIM+y!fz*vB>K$?xi z#V%-Ig(w?N$sR7;<4-`uhYo$&{QaK7TM~()SZ3{-Q6har+%v-I>n`8u4fIl>7e=z& zYb`hE74hszDI2&>_lv!!>7@7B=m4dAK`8HatU+Uq`>BSDI=yT$B->!sRu;onI=x)Z z?HGSfuM34~PvU+HX%a)ZV*M&dx5GEIpN2KQ{4si7O_tH%_z{CTiZ;oz+DWU2ea3X3 zUcRtO!oinnLc%A%Y)HzpJ$2xM>_}O~SDlm2oTN&B*U9i;k%rcnT`5^*e`$5^1sk}l zwaAIuPU76Ly$nIAntUy40F;+N{tNpHu$P$b07}PFD)fk#)g=FSZge`Br1wD<1(Ocg z1L4Mj@`a=W)XA+a?PhHqwz53;Ka}wt0KR=c+?6 zIs&uQuWoe@QX}FXnA?cIV`fu+9c2J=2?9MBl3>xnpz1f=tT3s=L?V;~Ofq;P-K|5q z0RAoC(G@)p+qQ1&@wAMLkY^WHkBl?>t_4r)J)&Gyjyej zd%t(1_KJ_ax6`_;zcMc4hkhrXVphT6^)UvyeD)4&{l3@jA<R1%+JoYY}<74ov2h^VbL6R$1{(~D}Yj4PhM`C_V8DVdhi z@=T8Zi2mUuQVz|e>G~+&9<^omV{wVebCEM2K5-S4U0EkT>%Q%xiq*ul4E#fk^Hm3a z?r^FM8RZOVes}j}Ow5;FPbHP$_Z*kqKI;Y)lxlu$D_Kze`kCyW$-qJi2QPsT$pZQ@ zH8uSNeGd+J|8*{zY3ZfRkAb>|HRubvy*}Xt=e$ zA5BIw1!CzibLjS+JKDdoGf4Y4mR25}2@R$@zbQb>Uj725wciEqo56FD3thH(0#B2- zc0Gx(=Q2B&FibA@{>P%Dzd=NUn%Uhn-^%hRg-tGQukk3=^PDKHCZ$&e3#L4cnJ%Q=unutKXYeaM z641DfbknoH#d<___x>2$wWM@;C*FN4!I>gJyipIFVjjyD0k{QUc_bO=Tyjz;pSQvUwIOMdO z9^m5%ijoO1qLB5%olb9z%D+wHpDeZR6d#*X%1IE_2kL_YPz|m7rR{$S-$gj9uwEiP zi2ym@195R8YCQ)*TEZk|g(v7V0x+g@JrZ%a5ktUjCLf@g$U^UTv<3vQBRHAK6<=o< zpM&pTge@6(i^KxNd<)Uto`T!F><}rPU0hE8nDTuQ^cPWsScwKQ44BY_K{UFfi8{gc zn8|=Ue$`Tq|Mk2;DM%d1orHB@8L|2nFCMc)DN+auWE9=UkQIKhA<{pdzH=R>AE*B8 zal`o&Hq!9JL!w9ievE{Ro*2BIegbSF_!(7!G78etZ)*$eg6xxyzM1*bEB`4$qa~PK zMb*FdjH_!)uD41UCBpO2>5JJem2)90&8aoNp9Fb6Bkz;3v5=xrqC#tHb)qhn8;j`} z*MD(WqJ%qEofu~FVk#x6gooe1LZTfQ>GP92vazG>(k%ewVN6o>%y+ z>B>a1ZuuI?O%gq_kw(kEb%ph(`&zEEls=B|7VGyq4dGJNo%z;Uf;1-AgXQ4N7Dlx4@b<*tJbZa;#BB8rBjr!gVK?9l}C^F zTE^Upa!1GGdG`;U)+a+QxmfwNX^)7{p3D@O5Gc;apk@k2>h*Ul{`}xDrpSPWs~r^y zD=dE47=-=_CIJ99!lm$r$lyC4Dr7X>`~m{oFeBn}nm~sHqs)B{2jO1>Fbbm!huz-x z^nV4;bI|N+0g*Dlk;xfWBwV(;sA<{5ea3XE`%F=)-&8fKPdy#*2hVQW} z${tkWmc9+o%<_D4cDwZ0!lB>PJ+CaK584{5kH4(odpyFZ!b8n|WmwKMH{O0a@Y_}o z`5gn9GE|!Qpf@Keq0uPgoZtOpl(-ZbIR37Rukg~ zk2Myb4qehvc`Qxacp*!9a`svKOnHrliiZCJuk9xD;A%w~4tf22nvB!{@3WIf^+NE^ z6M2c=A~uFx`e-nmm2LL~V4J%9cjJ|`(( zuV9@us>%EKt>NNkY1xfS$GJf>zp3buFmUie++l>8P#FH8&xScBOw*A{VOoy?#)sFh z=WkZXW49|b;JfvdsvZUX4gOiMW?fI2uZv{kQOU+#Tbym{7q{nrd6@iMGcr9Y;X+2^ z*&*}QBHMm@x#m(MbePEu2Zk!@d56Dj@7~X;+hA;UDtzMpNNM4P7}ZJs7df*Eb|fjX ze0G4D3ffTw(MKAR=38Gf6WdC}Y7#XZOW19-&)hrIX+j!2t)zkZC_`%ZZb=nReotsh-Oswab_e@$C|`h<SmJOuI0^!W6H``qOBt~A834c?{@pWjT23;_MYQ6@iO-cFC10A#Lg2I2W*k>KEWzD zHROIgUr>G05);3twQ9nm&}1r&xFi>TXX6~;!-fhXP*n)m3X6)f?lEi25;FxPD)bE4 zi35Ne;igJPnd*!ABe9SOo?e)B`R}+aHVezQ)oT?$zPz{|R*SZ9ogqAL`WbH*9EIpL z=u0013E2C41f-gpnt!K%KS!=32+L3*u6spD2XW|&>9;Y+hPWy5NHv%YuVD8M9z?6U zEw+SU?y(K?^Tu~iB=gdqSl(8Ke;5p@LYdjdR6=7D##wbM;$DyN+xrh#pS_;~XS$`O3;wo1sgw31ZxK4;k}y^h`a)@7^q7?QRf{%U#QOWrJ8ZIe5z!3PX5~6sgIg zwON#O@5Rv*^Kywp8V;e}o`H$N`*UB2sz@xch}ow_C4O43D??I>#+HG*yOUh0(){dA zF@i63);)`&>2l~Wyrk;7->}NKr<`|=A*d|T!iwA3%jo?1XooescAIl^@{x>7s7V(4 zdirnucq-$Tt28A)TqMAtBY5;FSK4qgnMqIY;w#YxjzjV{j_QgTQz&a?K9;iO4e!i5 zp8MdU=ne_n_41uf7NbpD1K&w}J0N)HV0C=O2_OCk$Ch4)$2afGo~~1#tm|MUx=-sfWs>h3ixEL zS0vwMpe)&rdJ%);L&y)j@o#T~Cd{z0nYMA1XS-Et7YRwLzZM(KMiU*jDu^oLw@K;%MWx`wb z?en>6yz!%sOcEQV$s*8LI=0>-ImtcfSPYw0d5iDmajUwuVuCXfYD7&TP8uI@_Ge*S1DKSHHn+n zlbQBf$eb*fzw*K++q*`0y$5_KC5>uMZlFfLJK#04=i~5j7fJ#=&8YL|vFSJuBE(CE zQ?At=#QCuD>{gDVqKHIX|A64%k*Rfk`VDMMt|~Z$A_y-!7~?o`iV`9_4bFuS1R|tz z1|8sG|8E!YH;*mC91pI2Gi383}g*i&D~ z6woAiW8MjN8Zq3!X~N>;_}C#hfl7Yn9DTC-$Z?I{%!DN=-~jNr~)V7~@Sw zd5aaOy;)(}6)uhZFOyB^<4^vyL_Joi-PPwPrjh=LU*e_WJF(xK`U|KpR>GLoC3 zZuqa7)Yr+-pfItTXYjsEtFNR+l!vc3*+_CBHKdIcK1~^8c9D{8laKJubfU2+4-3RpYBskV2oPlozoW$<;o`4Dx|?y#j-6pDAcmRm z*e&317Y+Yk{B~44+Xq%76-uT`wPAUJl@vQo><5HQ9u!)sSlHJW?9w|Q4T1GD^RA%>6CK6_vGAJUq!W#p-eXs`vqiU_a z8$W?&OMJ=+H;IRxMhZu^MT!iC2xxkG8miE?179x2VEBNq`!=+x0Ot%5SJEYHcu@w4 z9*rWgAP8SQ=Zy7+4xcpq5@WAH%!8s28%Bwcix^*n0<=oY8DlR3TnF#->%;`G(kI}H zs;xm=Dg6Wh94IGWAw-%r!{xx(+i}482?)cBT+>%%_8mpOn41L|Y>V%R{?bWwCD8J1 z-tXRej-9f!>zue-plPUbdfUS^*G~0^8SMv-5H*Vq2m&A`_~Io)IYd}9G3;O}ONqbs zU(%MO0&F{j$F|Yd3elBV;@#hjA<1cAcBsQ@?LVeEgi7z)S5QDmz%Zcx+IjOi>VAK* z^k1m)0DYrF%uP*w3-BAw!_`$kzoRBhdh;F$oM=oyT=;wXMG}T<-A+O10JF2Rapw+S zBQ`rAVJPLNBd&jN^T4kL23u_ifd?WCZ2^~fXo#?R1Y7WARJ(@HM;+9Tm~;Qj+OCO3 z1br1>=fOGhstr{T+VVQvBG)Z8b!d;*X2; zCw+`iG3ZiV({n{8n`?dP4ZFWDEY|$KYO6?NvP$Duex7J67eS$#SZ<^R?4wR*99zdr zVjw3{_P$hQH8Qod^V8eg_a@H7x5U*yoSMs>NHU;^GEul$vW8t%IA(TGz6OZ@)aV{$ zT0WL-5)v?(+@o}p++i8G+4<@XGq!C0mE%6!HvJs)Te6f8cX0KyNF*JWOj9h4ODY#j zx2xN{Sv2k%K zjqODC{`&_0xtyz4pD91Z!_5ufDzN<(MnBWQG(yddNZO+Ai+|zx*8a$R4cM%Cr(JsT zua^niUuKebykws~h}Wo6AiLqaMmznXXF&ON(bWw(=)R&q8zQzTLK z&GxvSQ#woSroKpf}6!Q5T2ZvL3S~_cc}Y>CPnFkcO%dBVJ8@Q8S-@VeccjGQA{eqMWJJ!9(eXT$u88@Wy%X5@&kZpCE zC_k2D^>=hx$EsiTmbdce)b$5Hxq7(5@FyxEbBNJW_UgW0Vj3UQ_q;r;XgVjRcvpz0 zDjGYZiBSh8(6yL`fWcyo`uDW`)sA$PMxjkal>fCj0;?eOep7>wA3dM#lz*WYgJ#Ne zpi+rOb3}rNGr^T@Ux5V_p#Xpx*7aun(a)dzalWg3?O`+FpuAFv!vmfptn6x>z`h@N z1HstE^996*ALNFj9Y96P*RBt^k5;?2PCz~i3JkVLegj(68i@vKEwmjSA7>{0&%)Q{&FMLI?3>5<^ztQ?|M=T7^pKdfYV}m0)4PxV zL-5?g{&rRzkl%H^7fuP^i=7x=4v*5dwD}$jUAe}pq(VQ%v5G;$Os`SDW8_Er$G;D1 zem9Cp3Kz%-?*N$~v3S3H=DKxZHSkP(y=$MXX56G zD&@GYB=P#9AeoUCcYQsBJHNCCS$0wts+MiInNvQ^cVCf+1$!HVjyUs{ z=(sQQEPCh65=p#Kc6*E&I)taCO=gGR>N(F01m@nB%3akU6=6YT)v;Wwaz%z^O$1Yi zTL7axnd&h(rR;v5OYt13`^3iecjEJ%+C8F|Ppf>{uSbatRnB~4w{K3~!{lXd<2+S; z(895>^=`6X((5MX=2lUyla1qSCXbQ@*I(Ggu4JGlF~@+HBY;J1f6xIS?VtHB=()8Y&b_dwK(lop zwP6uxMJNsMLbZVCK+wei|I!mpiJ}mI`@w?kb3Fd`Pruz5=IdJ684IrkVo5U?6BrFn z3x$n>l8)IswhrFI^%GvZR@J$IBG$I?QOCi8_b3Yy$L{3ity!x`lai8J;~YG<;lI7- zxKZn}e#5^C{+nL!f>Y$-9e=!27VN0&*lJR9hl7K|?va-;2^;9>s;a6q+D9jkr%_Ss zMM71#N=Z$bs8-AFlhN>epEl9He%-4Em_SVHelrYvqV17)QOL2LXbe;*UOCJU8>x(r za63D@baz`uQ@abq<|Av)%!!T`T#Ic>piy;vlCE7Ykdlg_?r}1kQ;N-TZ{u{)lO*w) zL(3o48F4bbu%0YFeOS-UvAD3EYIGGsq$aQ3 zE(h|eX=Fz;5*Mq!-ba1|pMytN1~t04w(+>+)Oopxua%-v>YP+>1Z<(wxJ*40x~z-t zUnMb6#My7Plj}%)5IeYsO5c!8YUK0Zvbi}}?yl`+lkQEMFWIqstP@nM*fnZr$8j{9 z{nTbbE7h}W2ACt3~BW%E~ z8*r7|-hFo=759Hd`oJ2=CEyFh@)h_;&w4>VX?}Pg(e7%e*khXUf2{0q5y{ZQurRTjp#bt)Z5XHH1*rB_ zX{(8wTjgXnN&gy|?dwbYEfxNE=l35!rn<*pT;3SB?$tnmexb>-uTM`d`+`MT+Q5H} zqsg#*($~n53*oK^I2|0kj^@{KSo>mc z@_hpV9v6;i{&PF3Kj~Ypjit*QUlteD6-w((YUedr)0R|t&bj2k_zt=wTk>}2L@&V9 zxy*{K51gEw%qZp=Ja@&7onmR+dc7)VX;`M4FMj;4$!w6A*I*5g*X~{9^X;(_dx znHiI3&KZll2J!81tqw4_GfLu`=29Ee>uqXVCv8R7)iG|c>rq*DeIG^%4o?V)H2bzh zCckYEvM;_BJz?y-rrJks-E*<>bAC)ZdtJYm@ISnDTVG?lfMUKhliNdnWlll=q@S7_ z=x6`c{Ltk2x9AfT^bNbaQ78hd!}45(Gv6Tg+P8myALHQvN7HqOW4*uskCk}}*(7^! zA$x|98M0C+M0U1_>@9`Jj_jK-J4?*6g2lixF-!TfxNstDDu!(%0uf|kVF8_PE&KB8tOITUAPC+*KIpF5V%_%w~OCEQ{)jyiQw_+_dt9!#P1tM zM)6(M2tU%R&y@~#IR~5llYTWy3gjVj6{?Mou6fC>7UAsL{p_=Fc{3;&PYZ3_4&@fC zii)N2^YCPaX&E$YPoprL8kzB>q~?QH2aI_!1MG&H1E=lX?>4E9gv3VkF|gPcE2NC^-1@>}$(|%oY$74OeCx}rQbg!ta7{$3o9%)G(s9sj1ma*j11u2T z$N~uyPcM5+_Tuke92Uu%<#~SdHBhpfA&_6hw5JdA6YGhdQVZIgunY~Y+3v6JZ z@dlhySyvd0r~^k$C&82t|Ni|8Zy8#k1nBkhN;e=LuyOmY53@$L4TDFZ`Gpy^#9$gG z2@JnwK)8Yy3Lg&-sw$oVeCBX%4<3hH_df~v9N9;FaTN%WPsVPDE`5ejTY>HSvGAS7C#jH zFu><{f9i|WQrkT4S~bjb(A2Dy#_y;GaBZ@J)w-8T*6la9j0_I(LJUnKFKV2g^+#Wwtcgj(8-?#h27#Fw?}IK-PYG9C$x;R<()zzn8+fRPNI9l!*}I6lWFnq zlVQt=iFu6treZ?w=&&e$$6;p(i3-|yUzbgthg7#%B6$LLw``W{+e&$n5(WaIosb5T+Kamk)Nl&Q!?j^(+$4viEJicF@Q6eYFNOUG{#xtQ* zpnbwGv4~A$nHr*HPC&5&emkMl8wvS$RA#~>>aSC0oa{7)%^&@EotwD)uK|BOPYFe` zojshbQ1<$@S!~$7ev;;uGTQcZ{_mx-I6tm)sP+aqV1u z>I~CsaR}xxu=;cgpfzadH1a)kF&Wt3q&H#VFDnD%C(PN=))1C@0nMjRxA6D={)I+B z6hykiENuv@H4t^S{6Yx{(^a;W!18|m8ULI8U@z=ZOk)@i}--57ieAU|d@ zXc`H`@8~#rAOHcU@0tDc_}l|mGjx=u_HToEP)1x(z1%C`4-(Z68oi%Fu%)kGBSs6` zAM+o6r+5FH-KxMBzMS~qkS6;4q{$Q&0nhKGLK42(SqD8H0w%%sFgM)maSRQL_oi!h z#z*ecVig5@5$GcUqZCwS!Y%_P<%R1RZGTQQ1Qu$$TwALg;tHSx&)!&tfzl>yYGLk82{;&SVhd6qVjiNEsWywx|{z|J}R!uhPn*sif z-M%DwY1XH9f-@N+PU#PFmtprnn~UL@;P}7)LSRd0i9!GXdd_2 z!NycQJUFO&z*7;4kAz`&%UJ{)l^9rYV2{7?3ucfXu4SR^;HA*l$H9n;(_d1aDa(G4 z%dE;}bWp)AkH-?$mF@B^JRs|7zXcIB)>C!D?XsqcGGb#oM)ZtmB4Yk#G zw;zTw&$it$jvxjusZ%>aKBZUhO3C+R1Bn;{tXY}1!2nkjUspHH91HiaLnAa(XXYq;@64w6Jaw1B^QD~@B=!hj!WW;aS8f91iA>^F;ajB1Nxk$ z7}}i#1OPgC2nMbmJaT-0|4Xn`R<;Z*CAB^zWN6Q z1!`*D=ootV)PQJh?MMyqYIJcq%uy&5>INiD!t^!(2!lU1JgEQ%$&mEmL%?Ul!NXGw zsMVzNf9Qu1(8ic@@Gm*JfT}-FA^|l4QNO$lA4vM)V+a@n^M9t-V)nD*{|n(ssow3p z?(%oFXKk{|_WW?uy7@r22zumb2IicUd`TNxc-X}7Z(J9Tg`aS&(8y!0KckbCG8iwQ zbMYzTqPJb`BGZ5Jqz;AOgKG6Zr4h!c2D*A&^Ju49SEC_)O%%@=m^-N=@Kh%`FihdZ zR~3mb{#0RHQaGQf!xt81+f9QZB&WVc7#0?{zAhZdjFG^=7hn}7Bov|{kMTN!=`|ux zME&YON{}gwgQx|ga7($ft--?~m$glB;Zfil2K_H0@*z|-Zcl7Xg&lIAg{2*FEIm`K zyrpx6DfFP*{kf3zuZgba-@WQrgx@~YiJ3a;VR-5`tPvM)I&3AUdNQHr^FH2D(KsS@ zM38!fyp}8e;4}U6pu-8Sg2@!><$eD^jWg)jU|lPBdi-3=7}9w`X9D;I-zCIV8Utr!7UaT10SQ5RpzyWI zUA}l8)o-~77Z<$44~Vqyjc1?@BBvF|XxhHAmz z5F;_?vm-4l+YEV)%^%vTlcfX%{zA@t-FPBo%EL`0f_w$JAs~~0Sb9(Nr8cg2Rw`29 z1}n>uSi83qP#n<14tTI`$!)*`*wt6%$@H&|RtNc^!&)iUqUeGBd-5$?7l5@t#_4N4 zl8-w^sYbmbeqX6fip`>)h*v_fbC!PRwxPZQ`!^D727xO|fs6!pMjEMzF zFlih(0rs+KO#@2%z&Ffn%fVv5nVqu}fw>Z|v1j zk7RQQdsKyfjp3R26Pd%<;qHJ!xpiI~XNWw}V5_(TvZCm#=se*60mRomU?#qOdxH20 z$VWPP==;T)PEJp)N9O@k{s({rY=XD}5I@H(EiEnfD|prc+Lh5WH@dtHSSpx`!4xlC z6S@s=GvHry5TF3jr+Ykk?6Kd(A#&3l@|_^LKnU;)ctDta^(+VDGA#1_G#?Z3;rbED z6+@)4Q3_FXJXNO*WU+lb5yKO8*DsV9Gi7xto^?6O zcKG@gyiX5!4EmoO2@2ten7k`D>qO$P4TB2_N^!1W5#Zv!r4r=39nX-(Ln78% zW_~X!mPw_M>Av)&A>PuxH@QJ+x-`6^6L+86B&vr*9!BY)@E8JWNo1D0zjk(7yWOi{ z9IS{C4oGWU(+nm1%_yb$F}8o|x4)p_)EzsCB>BP8YSWWC0}6Y=_Jc=sUBM(jpUCi; zWS7Y0r?l{V0Tk-SsMPhT98DAW4@jrV=Ex-A6}o{TN|FLp0za z|DU4=TKn`H=l**PZx0a5=dkD3)zx*;UVsN3z`aGVNnKoAG+$o$fVw%)2wHZ5J7@$n z7~rPW5XyD1S;C5x4n=^l69HwFq;A*s8gNY!n?VLPwmjgW(3M#K8)BhSX1dxA-Hrr9 zDU|eDz!U@QSpq+rlZ4S0s>+Y&s?n4;tC>O=Ow{b`?ADJtqyT#Kg;56}P1)kKXG<6J z##ofwk~dCX(-r>ug@P_ukl~JaA3pQ+k>ou2Zu=^Tncea3q>tHuVp=w6PMTftcoX$e zEvW%kFpWdj9C^Bre6J6NpH&$b=QgB8HF#rd>o?5C=W)=18w zcsWHym6ccU8s+X1Jj`7Q4WrV_T~SqYc};9;vqur4ifAMi+i1ix5F-JvqY!+2>Ww$d zm_m;T1nT&dFn*8|&&1PvZ5oekZ5D(JF01N=F$W;C@yp9{xAt&8QsRqHdkuxqowIPOc+@V9o$G(lBR z>&gG#@gdYlhYUYasC~9+rI1U(#)^`2G-qNgQO2w6{EZxT&`D|x6Gdn{ZFL-y?CJ|$ej#^PJ*eJR67UrukSH8n?00(*(*@B`; z@Ud~_f8caQuqj|J2g-d(7RJGg2kA*d7ch3(%fvc!zx^SY}mPU?2ly`%O|(^mYMbrc~HED*rRsw?Jp$ z1yWdm4qYEp%m9--2dSl~H8|-pkePG20{Z0V(8RO13wBLVu9<@@6@(4IBAvlQ17Y}p zeJv))y7k$ArnTGSne59Gy%}>wW(|~MWf`*&I2DNVA-$>D-f%7~tyaQ$le|M@pXe$2 z`A$kJ_R>7d;_Xt0mW{``Hygn#1~LRwr;-GNfy9~bHE{)PI48xdOKn@Y_{?&f&=iQt1Td}4O=M+i` zI04iW6hTZwSiT4cRt+PH@dJT8FFO@Rik<^AHL>Bf#H?@ex02NfuF(+15Gy3D5v#@0 zuAl3CN=mhh{{4QM!1RZFdvg?H=WoyaT;*x5;Pqaj_0%vYt!nZWP$i;7^wKI#KhO67#J8>S9bjW9`EfL z16%!w0SSXa24>$JZ z=!aKfGLTJ$*ub8emqd04q<=#vqUxy}hDNdXf4}bjN`+|dMU+RLQxiaBk!Nuk6?QPw|Tug|lV8Nyij>_dA zuoOZuro7^@ABrtIb*_a74_;b&SSazA^w)P%hCoyq~<|`X|K5WE_DDHrW zcDR1%g)rqS^{{Ia;?#!pYMTQJEQWJ$9%D1}Y3osv{Jb8(5}!DFXXUKlO)ydLyYUmA zpQ8b6H$2$#VS6Nsr}yJsHb~>pZXGDAEjtamaL;N_LpiT(@cP80HMnq#Mx)4%i?s9a ztkIVp*cw(2V)>gzjL@oxmKir}wS@08dMpli&Nis7M~Kke+TR(gvEJW9(QkW@nOOZz zDGE2Tq#8Oe*m%K>;%HgoyYev1*(yg<>3DH;^IcBdPyaj9l})`;YGa>Fnh-4F0V7G6O{n!~f)?Nsy}_FM zsEW4zwI8e+e{S}8v?=U}vuw!VU7T2bQuO7Iq`S`aNefPp3^_}tn3XOYaQJ%d@(6xqA~H-ZpfBmCt-!_?Vi_Xoz((oo%r}& zwRNs1LVrZ+Dn-{8^_a@c31eS)l5y&P(a_;vdq4VH&*WX{l=Gr-y?K6k22tsb zACsTI42`^jV!yBu{8(OaQKB0>`X-{s<%CHYj#- zb%i7+gI_jQAP5)*;-adm3g%9jl_ai3nO*WgN*~LPt&{0umb`$}%>SLf$bI?_T7rE4(JgBSOg9yjM2JLmZ%LsQlx!D&FVcGuRlG zctXKcs(Ag{bnHEX$FNo_#zd|n1+!cD5k-m;$jvk zN^Sc~gbAZFt%7pI-k=^wy{ymTiafJbP4b)cuGS_hdJZL)b=jX{4;_MtE3OhHTlF2d zbLF>1RyY=R=T6Eu&Zu$KM5!{wwM)L~72I{a^X#{lxxa*KSNmM6?qPoMM;f;c)W+wb zAY;@ldrN}KrHt#}7S3q}{fvBCv4ZJrO%i_k>)zgTh3gH5_ipNxZ;r4=wpVjqh|Q%i zp~?>5gvMN)Xp;pPqSlP1=f0I?HmGXV`$!(N1Tyab@-Ld=7B=MfJvExceVXjx_I&{>^NZCL zwx1PNn(t`bmxsT$qv(BnD<)go?U-V*$<*sUA2o_~49JpUAV~VCdCJk&*+Mko6R}%N z>#aw@u1Uq0yW_6$SC=NUUo;%~+nv3B6@631=(?-TWV$EE;Zf0{Ysf(P+y$zV(eY)) zCY$_4ern^y*(53I-)s^4%gKL-P0{B5+M9={xjw%}?~AnQ{~fXZ#j&H%FzH!>SE^05 zEBkjd_-1IxtiP`6wfVOs7u;<8^;w&OORe$C>L`821<6a8=O@Teb!0a@8LYe_F^^|k zR7_2jbIngzruo-+&M%S}*60^>>K`YKtK6d-E%NT}-<`#Lii;ZaFP%7?dpYBE`*2-d z-2|1K-*HJf0%EkBSQ&4y7hpUBcNQp9o&m=k)cYV?DIBH^6huj<3p@^-b&!RGdXP}K zSQ4Ura?SpigM)+Msf5n?Qot|jq}de007wUJJ0%P<^BN#;Q%b=7dq~*{&4+g z`GfM-Qs0#U&SqZqnX{zxXA+aW`)~hu2EU5g?!ghJ;_^<5G-&3{O;o&Ylxc8pJFzrw zo~4#c*mP!gmca8Ap^mt2l+N3!$}#p_b%x_UKhyQJU|p7be|Ckqj#x5-V25T@P~-CIUvG&pLa&)+z^9zN8<0yNkaH?Un&jY(89eY%*uYz3baNXbt0;=4rOE)ZY(4nsC(G@R zx(mweO$CYyv>sS&qQ0GCc3+Cu8!yR)oo%&L9~e|i_r9^BXAl)V0(nSQR#su5D8#`7 zds>mlg%-+vwj1Z06gUTr8W$J0VX9!;hq4=NnxJ&*fpGq6U@tmq<`$d)s(t|jr$QJy zFa$C;>FMj5*3zVXz5tj6DD~md(FGW&_gW?n{R4ebRo8bwy@EEn!y$b7J`-FMFxUQv zk8eAa-F8k1uUZe33RVlKZ_r8pq4(O*_|nut$*E?jG$ZrQ!otH+(*mF|KsnPkE!iw+ zDq7)BUUN1+!u)uHci;YqACokt=J_S&Ew84@Hua?Bm(3+6FR%r%nNoVxe_&We2nfqO z4E$h62zQ=-@zZr96rzLneGR_4!Y$L|E&{` z@!+;K=Hz=#mat@!FhvH{P7OMF7Q#BWCM6095`u4W_?S!j2yrf#1|t(D+rbP?okkrU zBOXb0@FxFV1r?_(n-b%Mfa_tMuUN2fXdW|(MObMP|D21zKa)wgp53fxp%K4zpdl}B zS{R3o!Pt$>1oJXUyz6$iU{<7X$iFB+ z_x#b=e+%^QnX1}}Z8qH5xy6^ZU?3oEwbYq(!g~=JRQ<~bTBOIaot|p4qXy&hQ~i8! zk<8)DtP$&*Un+>OCGC zlEf@*tdfDqjxj?+Ocnj^zd`>sSJqiqr<*KcWd1t)e0?Xe$G5VV*`H23XWGMKmgc3~ zqvJ$29Hai2YsdtDzm4{Dhz`2-@I*beOp?E#;$N%?7FVvr=7Wk&J|jlmzi%5K&s95f zagO}m%9%P$zWFBrlV4-9V&PnrRhU=uaIYP|jOc9jsgC+Vv|E6MBi+~fDpZjAbEW0> zn3dB%FQ3hX%Ua+1>)mpLH+4X@srfUx^k!p|G=1jYr*jqS>fU0JMG}*f@$GUMn$Jfa z)-itb3jAk!-|c!~FgSH}+}YbHuWYB39E-A9&W<4wY`9{uSt~)4`8!}X$pB?dNz1Ft zUv^{g-zaan)6Tc9x@uP3mt%SBrycglDEkf6(iX7I&n5U2S8c5x4-Y>a+CsryEGVLV6q0^ zlM0(TEy-Ixv#Dp*F!zg_JRNKSrW5U@2gMVlm90D$vjp(5rfCM{p>N>EkMdr!_O<@{ zu=+kpEF^w5Jnj2(dHHw%a0~S$O6a}*XfveX;9#IQhXlSYz>P<(*+}honl$=^B>dD)xf5WJy%(*JN62wM}*|(%-@kW&)j(b96Pcf`7siv zVj;ygMUVZeZiRFEU|9;?S>$LtpWEVN-Ap=8i|I2)ilh$H{8?Wc-{ke@zn=OAYY7WM z96qL(=ushvg}YV?<+y*}VgSK5GN$md-Op#2tL?+BP}SRe@}b-NK{o* zT9rZ`(BF6BWS#pjG<5t0MZ~n&)6TI7bNU&fdL8Am>)bQGHW|YRG9=3*L6&+RoNKs} z^zhP$hF4|mo%d?w#m>!2i~z^7^&HwK5`#0{%|g#LL=e?qBB&z00z|FAP&D*c?F+BB zU8~EYU;umo;4m1OK>Y&#(sP){6GEW$G#OH8PESueI;^kimpmw6gt*n^B{+Z(O0jWs z<|Py4E$RS=sP#c&6qZ}~eyK;UfbpOe$?!@X5ke4^ftDM{cC8Yl<$m_|qAFDx8Rmc~ zE-!(Q8Qq5~JX+yw>FJ54o^}RS5jjxy0g`y~*}ngcfyPzDpyU8^7V1VjC%Zp(_n>b=jDgo+Ki<;@J zs}0{Dvtu%<5*sBm?TnMRwenD4Vv=5YCBJNp9W2LjyJq@1HWMo=a8cGg21UgUq%5_> zVSz?e_@rFyOnLcOt_uia)k%!pb;5Nnr7RSdYiC^vQdPiN6gpQXCdOp6jKIYEQCpbz zk-cNK(N9|REKl`rIWeUiCQ(^Z`J@8gtJ22%S26-6;uSQL-t=SGe$f6@F87-6PYC@! z5nkqRT@EEVzja-PC0_+%#h}R{He%D?WZ}Zt?9BN&wow(t(lo+3O z%kY!so)4aSVSM$%{d@98`_$64d)s2?bhrU7ITTvT$x&=^$Z4*a4F@r^sq+iop10(I%&_VYF0Ybvmpqk{A)j)e|Mqy^tI2& z{#kBJ#MHc*Nq`z@(H-2$dh_m{6bta(pnU!Zoq=eDEckCWXX?<2vQ#uQ65y&6{{|Tl z=vxR*kZn6CwHhG-W^Wgm^zW?~)Zcr64F$3F0oVo_>gv$F4B+qsD`xN3TZpc%g(?uB z-bx-eo&c!c9!0%B$IT||{^tSE-F|+45CUKds4jT0(Of(eQ|pWOFg{ba7ooFGz&j|K zdhekq+W86A0dN{@fp&PcQVe<;W(kOhqN3dLC{x>(uLVN+uKVx7Y;DGTgZ+r}slm=` z1}x9(mhprKp8k(mDR*AVb6Md6NI!}&OWf_i$1w|MtoDGIgdsC9N>EwpYUp=|8bZd( z1l|Y>49wKoR~6!^MsY@I2hs1oGr0}R>)C%QEOZIY<^cK;^cPBWRP6Jhg8FPsjn8OH z7%EbVvN zXY0smoz-($xv|}qIOCm1qvPy_$G<Wj30vuF8- z$9uo??n`mupfty>6+wi7pwX;H3b}VixUg#0;+>K7mh?h#dKni{o!Z^a&Wt*akm0p1 z^=meJyQroA(s^d1Zj10qojLD=FS%b&H5yYBS7IdGFMC-HSIkxkvEEZ!9@nP$zgW6y zCwuAHAEW4MBCS34@%*U2oOh%7OTm2S&4E8@GsVB>9u|6!68$y-3~1Qjf7($D9xGpw?4z^v?HZ%?nPc-ce}1s&slK?r^XoVpH2#&CJ_}r zIbfdT=cBFsO@kmKzCKRpXUEnA#oMt?9sQ1%U;li@Dm>iLUvWm!kzD*N?h`~1?ajAG zPqym>iw7&WxUi53Ophz<=Rx07ahmw=d49AwY7Q_Ve49XWrz<}E_~{#ZwGq@OyGk=z zH`A-aiJ$b@rbn1#bwyVA=$v^3bValxy#gS(0@xoAM!ta1{nBwBdU|>w|4r)M3YIzH z5TQeUIoCkV&11y0?*RQlV4-7}UGxeEughq$kt+yHK(2d@fB+ry>LA&ytEY!$DrZP)Oe*z%0T(}L%dUB!mOSpd+l7tIfp=@Q zg@IYfRA&7pQ#;jK;rGW{S~d9cN(2GHuAog1i^Aul!c+`a9SV+ACa!QgcfmmzJGVb` zQ{cp}3}oD-#+x_2U8_AiL&oQ+rWVon6ql(znHV#GjLoN9l{_~ykRdKWFa&#Qk(wDh zGBuDW>^43|KvF>b&xcG&a@f+7|Vh;pE$Zf+iKoxNwIZzWKOCi2B ztRhS$Zl?apsFavmk#iFR^U4o7HV$i5d1XpU-H@KEC8WK>TC`5xr-dk$E~^KS3l?*P>;&$k}1JoNGC~Ib3ciY zA9R-f1TrtC>@5ED6d9!V*_f3l+FW)GP%>|uo;*>eZ2q&|!Fun*mNSXJ?;-#Dl*|bK zMxV>l9)htgv4tR#4_s0KjiDWM>A&jS*Z6Z9`2IV4?oKINgXt}pnL#XI_J&M*ROlui*Os(NQsMk4Q9%t zCnZ%?h{A4%dnHOLY34hqT}(l>0g7LAS2Wx(=U~6sXK)kW5pM#S=mjiBFh@y%b689E z932@2R0lMB(;$#w5A9z<{ujQMCy?0+{RJ0rT%e*N{HsAV7)0Bn&;cKiO&4{(0aph& z)ys5p+5w}nI!J(c5q3C;&7e@lo%srCz86>clIs>?(=3HX=t9o9cGlgTl89`jd1!*}BJHOoMvkk`ji_+n|Q-4CC+$D#MqyvQ0F2 zaC<@J9#FmfyOPB|Av2nXKlw(KA5c|X|m^VwJ%Or%;mOycGV1Rfz_ft&%Daf}JX z1et;0^}%kTipL5dyF-%399S$meu1Nau>FY{dEHr?m^i_)EvJPoi6HpHBhn{6>?1i& zruM3(x>V7^@+CWJ>K*bS^&h%Ze4?nos%gILcqC!Uo)G!yipUbny*LcKBnDbT_ufy8 zxYA#hg|st~xV$vySvOuX>H{MZBp@hm_qHl_VAOLl@1}}j<>L+h0ne?d%-C=lTH*xi zOv}Y)ytaH(R&%^4+yIki`mc}uOQ-xiF+~>IGpsSSGVo8hZ%MJqTx_q%>h82jn@ai` zFS+^_8StL{n=51z_gtS(`ZeS9AL2{!j(Zz5U+H~+YqWWa{iZ46ME9^sOzKf)hPyfc z_Qe+qf3@hlMd{|E^QqHu7t?#^@r@_%1T=#MLk<2c4e)!H|HC>ikp275%40?6i-S?w z=^I{Le}Ylf!e`OltLM@v;D%J$!0TZzGJ8`qk%nFb^K6J*_pA~*12;MQ@Y13^f)uOPwHoXf zC*eXNCk}!xK6`i__VGJ9iJ-4Xr(1Neryl)-V3Q`OSBK~Fl>pE#!3_gJHt_ToK?Xs> z0vS8Y?N4t?dsjimGzg8rJ!IbC{v1F>7oW`6#4u<yPl;beE#;kE&1l7#!&O`kPtsym0(yRO?GCht|?$H@`gz`!WLWKw&r{qdOT zgX;A=B-lS9GC3GFitvVgcKI_8m>9wj26I*Ubra-h3R3OeSAU(pOs+^B6iNIobMuj~ zq@<*)wY&Rj&6Fx&n6XJUFx2g3nwz~ax27YaK8Zg9ed~Pkobt}5LxOPwf ztyv?@snvGkVp|Z1erYIfM%N7rZvO%fWy64fkemy0l3L z+51V}z(H|^?NiqrbzFQ#JOv%{{GVOaOvV_W3JER)m{!tD zMJTh04XNo>QB<04_j`NR$(s&-<{kToEAK*!WqAvE@1#SnWR+%n$oJm%Kj)(UA$9`! zxcTRAI$S%VgI*r9ryd%4EUQ~|#^yK1gr8Tc<`QwfPrE^7wQ%Y|7 z2gDWIRrfm%UBgel2eM*btn=oQHhNF|mH!#LK1!REckG$-?WTGIdGi3jon0{Bd118Q z4t79=$3>Z-AMU$s1d&=e#vzF@zJ5~W)3rO&%YTX4v9tfOVW`nVaEs-~b@%=`#@1Dl ztM!xvDB2}gS`<0Ai$|D6^GPU)(+zFW-;YPuZu6Izc|Lfw_4nw>C2H}*T|x-3f!D3jpU0x^;a*e5(aYsCfm z_#8lf1c|x=6H2?V6V<|VV5+e9`(O}Ep0rWgd3&uZZp}~5u?)y=OKpk=c?HC% zcoh#0dV_RTQHfET%45$Ik3sd|(S}L+tMrzVLd|^PRAIR)JU*Qb{U%fKo3eAdtUYG9cIg;w$6Yp-<`!k!jp=a7CoY+^0NH7n@p&r zo-EI#q@=vdw#wtA=1k;dK(H;DLNCkZS!=s&9eD2u&bS=7apuIS@PC$IYX+ftngYSf;aNi-m3cCTH3d6O=J(gi<+X`jXjf|=Q!2?} z1Y$Z#%MiLQ&^{M6q#ozRjTD%O{=(6@FuG(+CC_y5j3HL)qz39oiYt=gc{yly6pLg(SgGx=}Ce_3J#(00^)9I>X)6s_A6AnZxEbX#k zA5|uq$VoL^sTIsGD~;Q4wqG;%d#HvA2TA{T`gN{+H#@{*Mcm)V*THq=@!M?~&6!bS z%W5OrUpEQ*Ba0|6Fmks8zI@zY6zY6Y&wIL&{lR?XjMgJG*2QzbjEDR*dPn@@_dTU~ zol1j;zon%N!z3HZpPanMMmS_1Sv2oa42bV!aZ5-T`#6DM4iKz5S5DN z0F}4`OIKW6rnIEXawklOBh!*9;pX5qg8cV7NGXPlp;_CKZT{pljQd5tuQap1@um|J z{jH>8v2%6*vL8N0+0-P1h3o!&jOQT5y1T*8EL?Lwq#V6!iVo7KF=b+!0LJ#1^}|T+4RcXKtuMa&CQo)7(#e ztKUD`tGF9Hdltbw{Y^X{yTn-e1`{Sp0HzC@JK?KOlk`1lMXv?qM8w4x%6ir;98K%o zQ1)Mhxj#9@TQI%~#Di0SeFssI=U`e$6K)T%=|pe{u42j)D-p}-U^39*y=6f%If@gx zsX5_Z`P1bbQc} z?mYIG^%uoiqL}sHhj?e5wy*Uj5eEIM)f$qGu*uV2`7*>9fE`e;jPakulF%6jb1U5p zRh*IS(neko9m!^CvDgGC&$~Qws{qpv6OUAL~275B> zzxp1wXqjK#q%jcxm+nEPo0YA6U0zusWbfK|`GeETY5zwrJx2*-PPg&p7f@8XU?(}Gmlg%>@IA=fP0jdKCStr#cw1)?aad zdnS53o6YM3H!%5~q3zvZ+oJ5*hkBR};mT8-39j4pPCuxvHHa;$J72DJ8}ONzs!isn zbCS|&6q8ySnd&4j-Y2`GL)Ly96K|gMqs(ol}32NLke!a7u;05!jZ185?vVah{K6r(S;*q9JNX;n6F8x@yJ#0FQ*ld z3S>)E=sD)G{g}`r9qDx&jC3)bQ3z$5e|q^xD0HkVW}c21pGnXb7ZH44a_jrkLCdB( zNjbrL?`EI7&4@=5zaMhBns4QL52?Ds`zmCKsQ&MN%dHZ1$)wpm%hWSVZGFe2#^&Ut zC55VpQx<9s$rhFO$qn^~nZFz3NiPk1Uq$Eh#jR4$mMYvm?HjQd&%5hi>cn=w8Xjmc zx-O~dJDSyV+(C{6B$Sm(R5+U!;eLL;%V}8lRTGC)(@f*k`XH-&`pw)Yy)f9uY^0o5 ztOvRfWMe_A-k&($*FW-|S`GEzz%iQ_PTxYfx&GGP*!Orm0ihq9$b%jadTl~PZnAQI z{K1u$CFa$g@-KA6G*0=f7nQpx$vsY!nyL!{X})`1n{AT)rKa9A%Y%|u-qN4#R~`yg zY+Ua*tCamcCAGKEH{LCEy188?bmK&nFOTj#?_M}5sA)FH>35}pbKHVL?` zXZ772+ub+)wJP#o3asmp&J9ml)vYTgZ@dU^P*WG^Z>E8P?$#|gNc@5lL3A`Qc*Bm^Nxr2FHng*yAS=N;O>M>L1W&jKL!xX@f6Al zz{Cy*E}A)vW|~zrn|mJp9?Xx(+qmv zL5Pft1Sla4@+?Y(l$16tIE(?Vd4;8|bU|uPd~%4Mcp#6SiFQ0GtGW!tu%j?>e5pBJ zJNaz%MU5odTYo{m>F04pWOErA8Br9G_fF9*gkWIfe#5x^R@>>0PDKUXk=;-dzqw3C z%G-!Q#y5>D8B7@82$-)7k|3Y5t}xv-6^LZP@5gqMW4zz4-^Y{8poSSPH?I-TX&rNr zJ5jWL*E!~Ztn{fK_N|1ErgykYHcvcM z(A0JLvi{Th{d3gl-`=8*ff*}=<P~%)kNp>7bT%%Xd@o{bRD(*rqa628&AR)yUw(H!@$cz>Ref~qik%7ice)jW0UiAIDk8$1t>y)1FZxD$OPo{cx@jkoD-$(nsx zQ#(kvZRDDivTl$X1v`;(jl(%~(0Kaz>`Ya23kVGQZa~U`Xc>bHl=^`2Xbc|gP-_7v zMZfTV-A_!k9Pr;i&*PQ++}vVtj=`mYLqOKKe+c2je&A~Xdp*EqLC{^Gso z!H{%o7m*F#=QH46Vq4r$ceOR5wqJgM!nosgs5*z2Ts!_V>AKt*58}X8mlRX*&1BcM zoI@D6FAQ$5Lp)G-%>ZQ~h={gb9$7to3MEBN5YJWj{-V8iX3Fzls}vENqfUNjr>53_ z&x{JKGKovc`AxxM*WdJF?S%q(Igj}khw5i_G6B0hwwfCeJGvClKdX3Z z+R$e%>7;X>jb11|^k};%v5JBK4vbt*REV;>NSPBIiRMw`<9Hf|d?bK^&c-SRgQO=_U z1ageGl!7%rk&+8NKFYm)+E@Ey)9`Kat#4^hrSa=M38<<9v7EMsO1ge2tJNU$^^GyL zW@b^36Nu{St6aXMt|Xf;+kY9JZYCNGQp(iK=&cwwozr4FcSbEcX}Ua338F0XOGx^+N)AE`mre~A`wcC>q7G}D)i?*%eA#n zB9B}%43`$lYR^Q3G@no9oBu_LZK1wi-q>!@t8mrS;k&%;z4yuu$^0l0V_#-wmuGKM zwYECzxH8~7{mE^VQF42S^r!BZ%fPnt2nCYo;}&^^3ej4GDD7&VKM6u=%Z8iXU(1N zF1L1RZFGQP*Ol{*0pM`re^THja2TNr@^0CW}J z_5ixv2h!iCmVgQLgKk6gVtv~=yUj%mmzg(d>Z}sJ1K?ovO_5&atE}7(-HUnRtD0cb zgj)f&^p_LNM#6XI54rdylN_{jqnlw}=SYvLYcXnISVWva@9qW$)}gGqcJ1Kfb^7cTVSUx~`kv z_jy0h>v@fPmS8#g6?T-cR0I;d4p^QTIjaH#9{R2|2_7M#S72i6Q7_KHu??_5^F?zlsl%ud^Y*D}uz8L`qIL$}gpSqds1aIc;5>xe_CmY%Qh4WK7|Oz78Xf}{cwidd^`Ptw8*;KOc2ZBrT(Oe=`Mx|HqW71`189hMb6 z0<611UA)HM-Trka@P&0bt2*V4Udlj`C6j7jxp+;fYD6a*Ss*={GAeiOKJh{z0oxNLCj>*}Td6E;ycSO4 zgx=Ak9~5CciBz_NBns$+xIs9fG(!HZ9~q#2TF{{Ohh zJpN$hYKWR<8VpRK?YR~{SDb$7@2!V}+CTBn`t;#V?T4rbM8o!)|Lu_d$ewQ!wfIoH zvygZ>`$nUu)xs3flC{QZ$a`x38Sl#OV%FK>#`pHx`KrhCp#&KbUFCjo`NZ>B;#b4p zY186>nPTF@e#M2e;TsbqqvbcJ{YTyi5<6}Wmlu+VfUO?p;`enu=K21`sedhf&!afg z%f*@&`)aiKh;|4ne9S6G_(<_H(kFL7+PnukI4v!rTR*!jSdZ8A3xFl+spOtkk|mpn z2pDKg_;t6l^oE6p-%cLDNy+e9O9mM`G;?CxQLsR60A86#eIr=7-d-HpJSMM?KvY?4 z{V**df&NV?Sou!9g#`(0^-o|&e;Y6=_|g>8f)uFz7?f&++b8X-$xL4@0uQY&x74U+ z7j(7Y%-!rGJO)k)w@$#%x{2>{ysmMXcYSL^)#yZDDj-l9ezU=)amQAtKubw4>0h0z zKd0eQPY~9K=dRcp!*9*Bh5DzrZH$E`qJ9&^ZX z_s$2is9!=h2pOHUuV1_N_XP=JvNBs(VvtysU1I2@gWsYES);hSiWz~EBYKGu_%Ds@TeBtqpB<6R8JZl9}i}H+* zx#r$FnrJ^CWS?!G+@PiR))oI=hKr10?s%r`^RZS`xISTeB~w`VDxfz3HBJ9>y(`}u zIq8RV8e54dD3jHzHnu;LfWHJKwEaRv^vvQ+@7?fAqu$n~ng&6qxjTrzSSjW9st%_g zaWk|YMQq_)dh9lMZ@s6}n7mk>xMEDNMn*80DUH6m)1%lgU*I2FlrV32wA1q%(SrAS zaIfdBr+!3`19gMQj`t1^wP4@s91Z!1Y3n3sDMPU7zvsU-9Y_-XqxK}P7CA|`4q?dO z&&-|5vDv;>2^Pi6H{xb~V|QZsgaYyR?Xu*QPOZwk&E9Z!EZOp*rQ_!g%l5+joDSjdEB3s5F6BYzaWrGk0j%Zw{{DfD z&Adl7&_|Ng^kIt+?nxO1b34Q&5CH7CXUf^WVc{&PLnZmz7XmW?003Cm1IR0YPDyMp z{)3Gq1i^j>O-4%J$uE2F2j9RX?gj>Vo&S@hBb-1c4u;kPej%{a?-Vn?&Eo;Q8TKN# zUWbrrSKssW>gsBiy_(+v;;<0p_0#LV&=LYyjpC8rXec2)f`*B|T4p&kBzr|K5Y| zeP}ELYq;`PtBOgYyUSs^WPzCZXeRVIqra?WXlpY{&7_;#bXmE-$YPZUMgKz3F+O5t zWsGuW#xU3lL2iAliC9Q=?q>u}nG11%{Pdl5gfZrd40o;w2?-VrjjJ^hH%%-7am*RX zfs%@bW)fSA(Q0^>6RkEIQXrx~Mlnp>S-2e z`XsIgRbgEiS?a}K-tVc1*;krTY@bChlcWgH#!>nSt5y+Yfp$4O8%x}|G0~y5OG6KO20+w>p$|Ur;TnVqv5tVV2`@-Bs@g`udQ8 zCgg>2^UF@N5WT~$ePxx7CQDKO*{gL1gwVBfhKAkC%Ocyd^VyBa$peFdCdsj+rtkas z5Ai~^WBt#M-uYF@>qgwnuaNYYNv|a~RxL%}Tzwk;+VL*N7UQRYWAgd(fwO~h9m&=~ zWL(dj*Rz(?AEc7@x-hfMG7jy z&bwDwkRrM8P;~75&EE>a>1NHZR*FN>G9N_=(n3(oxXP{*`4KGV-Gx=Yu9CCQogxA$ zUaO-8@XmVc)K5QBnTy0n*O3i-c^rM}JMk!H-j)>j=KMCu zK&5H>qNsWL;r3=X!K{A@Qa~cYh^sH&u~$)Az9sBa>d(PKB7&U4_)+t9$pl zUr>?8Gw-eGe0I3tz1!5(u$4U+ZA(%}eEKL|fz6;uK=Ou8P9>hz?0oXOh1#axodc_& zD|!Qtk3dQ=D*g?qAnb%C`T38kE5NHrEkok=Ql83y)z1d7@nCFwJ~y#G467$dZ+iOs zi5Tw2AhgLxcuP_gZ zRfgqOJLiXq8JI`rRk)NBRbGexpoMqJXljL0cs+g6+n9bhF|Vnk65NC)@}=x@bTl%3 zE?gmVuPNmp#!dHgF{RDIt9~%O35t~Yj>pdZaW?9e_K_@& zo!RoW8dv4jzXpZ5?<6f})@F=H%JqM8dOjayyIz~Cm~VdmHd6ehwy&-H_Kx+wZ9SK_ ziP4QHuIfSV?lYsCc;wYN$%A#WM=!rQDV|s9)sgnRlsU)05#m{#tQQr#%--)3JGt6m z6)o+!cDJ8jEG&89FkkL&TD0#LMi<=U>AN@kv19jgD|UJKp(REW4 zyhusDJEeRW)kUJjdS$z1!aYAjd<}|QizvTQ%@vP#+CTT~nfS8%>i7>6&e(1W*Y(8k z&1<~U-n)E~*N!jUW)kk!d#0=N-{fpQ3sechyV=eChzlh{eHh-ChuB?g!^rFK=So#_o@mo@hITk*rbjUkhsP7HCMj80Ui>?=~SYO9La+i~iO|kzclTE_w%&`+O zuRn5A7ZqW-TesD~UU^T#q8lTP{rp<+#n354)`P?i%u465e$42CtmP9Joq`kd-dtKG zxG95i4TND=Sb)RqA&I-e4KWb6 zq%v!3k3YlD0L+3`>-EI4?{epR*hi)byUNFYfz^B@nBdQoJ%fBGNbUv!dnF9~u-;D| z0n=45bMb=QA~20kT2%xX8CY!T!UM23v}$Xb&RC?1g2#e1fKJb63hJq_buT&M4CV`P zBeyeE6s3#;0uBfO#-RsH0e|$5esiaXW*_avPo#cg&s4<~K6&mmO%~WzK;j|!Ju55f z!b} z@pgUwet#MMDx?2Pzun_ln-!NU9JfE)c=y*>HTOJH|Qwj5%i4)b(0p3W$C=p zRA&7g9%o>cQp(x#vpzthHIrzyC;uznJn}Uuwe!ETYu7}Lf=L5XF5!FYvN|MG8?P=P zh5O%>{l^F!b=Jv9J|+aFG%Yz9?i(%T#zqH(qGl!sPG1K)BLj_)rt;$@*6~j8mfX+qminU6oXV~ls-THCrT*{1Ob*M zCE+7)bmNE97WX@O1ELU!&;Bf!bYzAv?`sBz2%-=%+`k#dwh4Ot3?+KRFJyx|IIzb( z*;O4A>2qlSVnG&zJ&1rS$eQj))o;!~8TQS^KutEGyr2MXiuTAN&eH+DPafii!m&no z2pQKrW3k*^EFM~HV>0|*yLga!I1vxV4M3m=5KH0X>g^!}ZsfcP_<6&Eyvc5ijaC-I z69Atr4gM@f2}#%^^w}EQK6zPa2>pZoG0DsMBSs^I%-S#AFeV%`- zzcDTCq}J!LX&sn8k);x+h}%TXyw-SE{fF=JE#c0wQCsA)B<}9{+y`bpY#qBtD~EeO z-Mu%fRoslj^u=DxX{2~OkZ|YJL6XXdw%O#jANSlSn|ny&{xILGRtjU-pIR$tF)P-L zdUC&?uf|{~a`vT8p=8D1FWEkw+I1PqA>Acr`_%Myp zsT=JrXwyG3^s-b)Z1XgV_HtUAtpD?0O4-ZZ+9B{Tb@i`aayhmr^uNk=$GrOB7sdf^ zd5sK-sbydp+@TUVdaXVMJq`t*b!0)oJACTj$RYfYXYmd8sRadd9euKx`~SoBcI(&V z8-xM3I1J?(f!vRYd+h$=$ zBte)hz>Xy>$~npjkcxPH%V&rH`)R^^_wErAKF-(N<^KWkIY=nzs-O73LXh#=57-ER z`bgQwzGCt=N~eBQsSk*;U=}?zG*r$}uc4u=Tmaa-y*&rwRuKr-ux{ZofGmWBgsF#M zb$JXsxx>@bd9Q7k`2qhrqSJuKx}NAF8o{qTi>Ze%>ss;xP0Fs<`bI-T^D|ygIc*GP zX#7(DGh1&mU9AR%5CBeIb~}^Kms{J-`xw{)`v|K4##XTZ!#Q?byBi*A%dwmoX~vw} zs}=MwTDKF$?cm+5k?>IV&o6m-DT2Mmmviu_!4C>!lcpZ;OKlg z`5r4Nje4svx6WhCeHl8Riz#0m;)B!EtHX`Y96lfMxC!{^d zfryza>~@Y1i^eoWjTAi_-NbTsG4sxka58Z$&H9Y&gsW{e>ps>p)&Ce^uxGKCd^$ry;!Dx9Tz zl>MQoNx$aNXe_AyI^1)u1k!j2f-*5S#JF}K3keQ3u+s94>keRsJX}Uw0)PS>TtJ(n zpDFbQmP{ubL-ag6kNFH~J6NPhl|J*Bt{)TmVqGt9})I{or zQT@%?hxv_}Ogkn2!Z$x_M`99^KN9=X{~Mls@)LhAV&E>tg63x$Tu(REh@K;ssy=DC{-U)v9?xB8J%he=2U>|7VgNEk2CAWCsVZSI? zWjbZt8a}VASoM2BBJJ8h1_XZ$te>9%3E~^@+V=PNL1+)@#$J%z@Zba!u3K=`gXkCJ z7BHWI#F3SqovTu@k973(K<)wSeA|(~XXy}reT!`ZPoh$iQOOYEsvN|V0AMi#I3J*j z(~Y6!-!>5V$Orr&Xk2amyUQ;jXBu*pD`3qACV_~*b43nMu1({|r~=e041=ee$!D@W zXq*e}B%PPGA)6Ss)G09*{zNB=$-EsH_^%oHD?PemoLEw6OEtxoz*w1`@Ox|SjS(=Ll>j^2dfJ1Izgs_fZqm^0p0##|j!5vTzRsG(BrJojUi zV;2m6iGMFjd;H-i?@XvmQCU56b`)IWTqmP}N? z_%y%82naeLAy74~0I+7oR9&ej<(fjsJ$tNF-b3WCqGVRF6=Kf-$Szd|&H3S!PJ=MUf@Ph^t9!~*kOIW0Kmxp3}f$CoM=XfpU_rC1X0mx{|96^f6 zsYIWH0yzv2VRS;6t;yl>1W3eTvY`uXmXkD?2`e0O@592ZgD)?;br^ztxXqV_scVsal7H528L2NGqR7@I7V|Ec4hInODQXPxbuPnw`@co;cuVA|#RiIb6WwYX-^5cFynKOjy5SOmTRQ-eOVic&L9d8MMWh-mhXNfO^lj6goGL@8_SZD%EN$)l$<40K73Cvxs8Ccpe;y_MtYKt z7A*>y#Z_pee0ZesYn78`OvF%2ka8FYVX`ZyW|-{*uJ~?rSpuR2R(&)b)`V!91O{am zEo{`fZtXQG^?)blGyw>gE{d-~c+7X%$cUs{>=krf?%(aTr8WI(%X$*|Wuk}dZi#|X zz}@gjSET!&h9*#mex!PIh`bo$ZGGR}WfZO)Y4DO$Y{)cUr^%^81RV)!_HnR9j4Orp z=Z=uKAKyGny0`w6I@MK%^%L5oDHJZvnuDR@s8+ubf>xGCye-i$NL2q>{$9&0{H2JA zSdka26Bv{6!#)GBF()rqG#NX5FuNrZk2B5NnaogT+|J2Cv2HI!w6-XtFT32q(0^oqaRPx5-+pnVYYCVHhV^J8Y` zo~E7Vo{dBL(60|#e6hSpQYTe&g3R&;OwoslNjIUlp^{)s2g)zKLfyXE8HPktuhj(5 zqnX_6M3}ku>O_A%bW;@-IIO0y|KTVW3wx zH2fs&3`zh%6jC4w6IyYCAxNAA<{CU4WNGeDj;(J$)*#0W8MwH_8>3%Gz+H6jfz9i} zr*}tg`S%_hz`swEV=~!#SNWAfyXGmLPiC4ZK18BHq^5&Le^l9|?|lK^Cd;3RoTQDa ze8B9iP|YE|kt`OG zIY|witwqXpQpV_dNHNH8HDjN(>3~gFtv?l?GZuslnkQsVybs#*418 z8_NKcyMqLS*j6+)$2b9hjv@?y$4^FEjcz-&N+qSrwM%iytniBco|DQJUqFwF&V2m* zZD^otJC}l)P6^o_qE>G*DM13g0!7fn^5A|+E61Q{Q>4iK##aip)YK)FE`2r_{CQS1 znDMX9rvk&t4y(=)M8RLxgFg>`|5m9*wId~6W2vEGNY=)g8iB+1xHHLiKf8`5<>)7+ z*1*s6Ar=XV*DV1^fUEyKJTw9#?dt}q-rincn7)8Fg@ORr(0(XIyQ{0KlNArdEyS1| zFJ?kHUKRKZ*h#=KPCO`_S`*&d3}we@GCo`@PtV~z^?kE^G_3T(5t-O=w4~d=jHOT( ze5sY=YcIrLfCO>Lmr0%@@Ca^%Yzi3cV4-f(=x$24{0@;ch;22p7G<|7l zZ@CHUi+=Z$dFXUk&mGU!u|K~#ZZHW+__kgudtKXPU7?v18*AnmDkjDJ@@TeZUhQm~ zPflk>g|m11bz&Hn;C(0l-z~y9L9R?egEL;zFGlECdddgY{H326bFXO=_l=ODT7-uy z0-IPT7-_&|%JS*;ci(Fc8;^%Cr>n?ccz;X_Bq*TaO~{wqV#^Am$toxS=Ye>}h{Q2@ z79wc^OwBLxG>oCCe4ky=DWw7uJB*O2nXDo{XqcNz8G8E~>WVB!5=CgbBN%CUOXecq zt9R?hJuMZc@+g$m+|Ujrw`yfEA*4Z=aR1D&=h;-PGzgtU4=%CzVi*=CZ#no()zX{V zwcnNhF0D(6UM{(SNL}w!H~$qQ=0H<7OQCIY{&%i;oexm)ycfP#x7A6da|>V|hj4oK zSn+E$A_DI3O6)p34d69D^e>|Bn=7V6^h#R)chdK3^p@Ye?b^%nc!*x*L!0mXp7Q6w=l__2l4V?O#kqco zbcbj?QDt}k3N$Y`KlTTaUe#ton^|Yc!W42-ujeJ2vB7btvG5{HC$O)eS z^@n|490Idn_^zRV03DGGV^U4D`6{FIZ z9Fp*;=GTXv?hLXrh|YX-A}493gX4+FQGRM{QzC4FKv$=#C!dkX2SY~$#KLDgyT1r8 zg!>N9#t@=PTm0+DR1b3dW3Z5)()IkkJ?czbw#zrHx9$Y8owbeuk@7kpd^#f0b1M5; zN51R#q${ihCO?wi)e+uM<{#-ImTtQfYPm$(NBc`meWo~UNFTgvIEj?W_4&VON(I;_ z6qk4C4vq`X4#cE>Wt*HTcauo6RC!#A!dv*>QGvVys~s+zBx;JTR&Q zvd~j52B`}W^su^RegNDH)eH#}m9)RVvYDgTf7qwj7`+J>uNPnb`k<%zpn!iRaXF#M z$7Z6IIuGxa#bzk>9V31$zY^VJw$*Sl))kbKm9vdQ<_}cDWfwJOo@SLOsEzzb7q%Iv z#g7M3?tj{v)X)H7&D`g?nEv9Kn~EdG<1M@PVz1+pZS z7X*OExx;6WT@Hb1OQ6LAP)3p=GcG!s8wl*sSfBm-2feeTnAjyDaU~b%Hh|C@o~2!+ z?+i|X(AZp;QV9SmakUbs;kfmKn3VK3g&o!;No3qM1L+Qv<*U$j!1EN9HvMDM&@~_b z#_lI^bAEfUi(=oHnrvd!_;Tg@^61?-*x7koS&eE)I!ra+Y{c=MF)9b~+v~&Y$Aefo z0p{x*S^vcHji#(bvG4feFN^1rk6?(iC3boH`-z~#&;+4yx3DC-;7ad7v>Ou~X;m)JW zM=0q2OJKi;_wGNVM|6Y?vRT>?Sj4U#?4W$j?3EyDPbXx*iA2r~Ju_of_i?M>?*ct1*Vxx)%lFV0Ag9%|{ADrS(IYUS_A)> zvf5c!OWPPbisUBaA!D}&7ZNDF4QIjawfaZ~)04Y|)D5D<1%e{Sn`omQzsYK>lSx8HZ&GH-c$GZk!GA(!(Z+PklB zZ1+FBsM(!rJUb-ebLp4HNhmR_=$kHe-yF;PFO49==9jVgQ;^-M=T!i~6^Q4%?jon! z5Qzbqn?NOi-UIrM+n_f%$fKj$h+0t0fvN})FJuJB&4GDq2S>17{|3(J@!q^7B!NSv zhrh3j;L$YgT6C&(ob?9ME#mKOp;iSDVi)L2Fk^MX=*OsZYeoWH5AlNsuU~Yifv$DE zFJ-*ec^S5Ut6fGnSLO$z`H$ZR;2j5=mFWCntA8yAMZE20^NvyN-}RN1N@kl006&JR z9hTF@r@w1ls)U)wG)?~JdZ0|KXfwZ-vy$a+KV7kV!T61=)Y?Y4lHyzZ*_ZB*dGuXf zm;19DlZpOD(7NBOuA5;>kMfncl3#ryWuu8axYKu6e=ln>l1D0^&pv~`^1ljsPu7(?l6dtW03VR7)&E$xQl$|%MoBRe5bv6E1u zGJ9+>U4BTtDfH|~dO{&said(3KGYg9NE@Z}d&V-4MXpnb4w>w}3!ivWU!(1bQ{Cx^ z?T$$hL8s7~?{C#!83xCb0pX}SejSd?v!cnGAYjz~h#Zcp-^dw&8`_JeOd3&|kuhf1 zH(DJ@GY+C%{#ReZT+u`L2*Iz#?7yWo5{HxeKPh{^7uL=LB;zlXb<)7am{e5)OfTTX z-j-;X?xDivLl>GZzBP3Bs;<7=(Ukbz=9P%#&Z6)QI-$`d7>zvIl*B{4vThq=|Ay7| z)nMUCzLRDcFC%hMdSL3zyeRl1-Ud)tNy6?{1c2d(Fz{a|Mt}NLkYnuS1!>&z#Ss6Dqbp}Mt`JYAQas52~=QcJJ;q{SQ=a=5^z^)hZoaUyh zY~e)rylCj2-o(%|XHb5+OQ4x=(Z23;^xUZ4gTyuKtxbkox>@weVQ%ldcwSY577f;` zIYYnIFO%=8wY zZ}>|3m{cG}6jAK0K$D5!B1e%;>0_!TpwvW;;CRv{!x(EjfZ(z+SYuO=VI4DmS7$BM z{)UDbH6M|H1=rB03>YYq&v!Xo42viKM`CCk@+DHp`UsVIhYX3QEM- zIw(^BaOWgn$oTk?4ON;Br`50F3mp<#-0}^bF%#QeW3&=XP&6%Cqtc^h`lQ6FYev)X z-r;mwd|+$;iL^c;dF8n>uk>3dwD&3K?cX`Sg~X}iJnjASBD+};@u=m!00nKnpdgmh zUQ*A{C##0~bK>nWNbJ{_%)Tw4sivlkYj%yBl}V5p1&%Bd3rYA^yxkzbIG*fXE}fJMU% zKm^#s$;!!*a2mRSkR})ZnS>TN^nsfh@T&H(m`+)4etFsMw6!DK1$(mXeH}kLZKcX+ z;T=HzxR3jTDn1>YcwE=Psp-u$zBZr-j$;L5SAz1h_l+FC>EHIIuwDrYLNzm+ht^i& z3wl1ZzsDfUfjt6PFag{C&)X2Xt2@s9&_~?b;nV)0ulRE=sMJ&Q@10WO6TKa3z85L< zMsLJK^6m6kCWQHGy>_Jh&9qy+h5tVnU^4a9xr>|C${Q-tlg6{BeFZuhrj^f+b5zF3 zeR_9=0eZ6^%6dKDL}f3aC#N5Izh>U~C}jn0Jehfr?V-+r{O5Ce3AF~d+4N5_rGh_D z+#cv#b~XsQZ=Mn*nDApnBI7n{D$pcRiInI&76;1c9M3#g{>sqB_M}SGVb)j5FAZUQx9-c}HylQoIc&@K85ykq zCDw5U1*AXOMK+yEaq5>2kas=8lag@}Y4Q4Aa-dZg*aenG2C&8Qtsw!{9C)475X%8F z+I_IvBi_35VD;|4{X4xyqBMCouI0L={f%a(moHz|&)&K`zXn2!t*tF|lTA*G$m-kW ztM{63uAz*rJp5Xq%-YyfMk)4+n~7;HLC^Hur)S{M+(oAB#Z`2*``b;Oev88zn}J2x z|F*x=fN;z$&`*AMre8c*otT&qs-N9IlPLaOR<;hdGL!j3I3*9cY1KsrN`FmMa`>gK z|Lrld5U_gK?6LY`+rPE?71zrLsxaLNdmU7A^wgzl_FnkE@ob=7OWivC8>ASk%Bn*g zjPnMy%2xGp&|raN?Z_WgS}TK(EHNzZF2yCm5L;&{MSx|I`e%POh(=Ln6_qsrVNU$U z^^1%Ur3bpP>-W{v$I4d8|IS|7SWi(+;^n&NpdmJQR$s(4U^1uv*EPJVOpJ$%izltr zuRO5!ednPj9Ya85WaXD2RN}xg`&Jzqrldj3%-(9Ir?0pv@{zCt=H?ktX)s$Kjayvl z(ZBDfA^PQzy=-Uf!Y3;~=<{i?&nW3F0<~4`CuPlStE)K^%2%gi%c$DP)7pb-f+HcP zY$nXdJ$pp)T4_VjCJ?$Z1|bCLADo^Jd-vp%-rfCuiB?10@qATS2H7Y8KY>w(QK_1s z3z>zsij6Xi!>CACCQLJUA$0@EoDw&}`ZINyjFSa!vVrN3suFer`OA+zza|sRcP&J` za$xEOh8<|}B+M}4fJtym@Lu>|+3G5s!tl8(`8V>d!i}6#EHMag(H;i}1zp1S6#g%O z=mQHMun{=}wbou8Dy=M_n0?kgj~ap934RGas|k7!Lb7jw@ee%n+ZbPPHU-a+8oRMg z!pO0_jMuj@TK(rxuDU(|!P^wXytm}f>8tdQA5S(5O9fueH9*JaZ@K98K-KhfP_fEY zG-(xz3h};6J)25+=ma$(p-z@8WPg~!^69a11nHduqngeFofM=8_=Bu^CE6po$MBeN}SticC!c)W#0!b zkp9Sb?i=8lt zlHN9=4+*u<$Py8$+9Pnw5EPQ*sfVA$Bry=Z4+x(Z^UB@p!NzjsvWa)${rYt0OY+ys ze4U?BCg0unCcnEUdr;1wb%t6<$7^F_T}@57AI_ybY4l}X@#Sf92nmm;@(}xYU(+l5 z-bA_!TPA0l3u6|I3^@|Q>xn1PkR`6&ELNKc2kyhAz^U9{`oy}*!LJ|MTi}OXKH=HZ zN{`2i4gQag^6#di(%%4834)ORaem0>zM)yOa2UE!Igau2-#Ca$iCLps8 zbJ-C7Q?5)wy)|A9q^D6ErqqWZkRXb>XvPC*OnwmJHzgUqbq__xtr0cPnyRIF+; z*{UVPAmlHyh}#^jbOE#ZOs}b-scDY<@cM$2BAaj@=AoG-`>0UWbTO~e#*=^J&DZY3 zO3XSsIu0lTx87JjFlF3a_}}0Y-u!(h*$c~k3)gFke&O|0pS{M@jqDHC@W=4-Ij9T& z3&Ew$Qh4CCLNW18Vdn3dZSzGBsg-M#Waex_-jUGFM2WKil=SnvOqb7klj`5$3SSl! z_CI80tsXDi5=mAq@Z58ZI^=v#g^qjHDo1vw{@-Q{e24Vz;XAtwv6sgiL4A|TENS?E zHcKbgYO|2~PiXv=Fw^?loPSbX6q>z15z}B*lihckK_33hUD+W|V|2L5hJQ0(ZvEz?wj250PSpv=m!j4o*OLir982K0C zDNYY9rh63nz4v#JFzG!F7g}Qf{#8~)pIo$Pk!L4#j+mj~_mk~0#w@|kcF_9ov8e2Fsz z;+9Mb=jc6PNiXiQOP+UW^oySw6DA8eFD-wNoP)ul*iR}~J#7jaK^TO0U=sj${C7}X z*GO+1oJ0T=1rgYRtO6|ZR0Cbzld6;Re-~y) zVtzBD>+|zkZ?`7|jw9l}n;m5DD(s~+zuw%UfHzQr^D>jhrz3r5=b^_6#~OKWlYA|Z z0wB+f%3}fva?a;J{;YVC*Ui^Y?t45&uOad$f1^vpF=^~*?)b%CoooAlU5qtV=9vd$ zrE{A>v2Uh3$i8*i{MK9e_Ve1igG8^$q@F23*|wPT4&gebY>RbIH4P5SXfAgJ4?1Cj zN4!?k9$$qn9u7^+>hp57zD{yXoIJVAx;kO*KcQXL6$>JmG#uga!%@pZ!lhUUS@KqI zf3e5E=ukNxyh>lcXU2{Z5Jqe}Y?wEr-G8`g_hRy{z=nwJyUI`(or3ZwiQUyy-F%E{ z3m8E`GAF*{w8fX5A)gYqbh$3h+NX4S0>m-#WHkBYzRlSqcOTLqxf4G32s~gyU62xP zwlR97BFT&=x7`nY$3qUCa0WGX_u20pTb3#K(~s` zhJ-9z{0NyoCg99|Fysy)+B^Lp2ZDsVg2fJS!Dw^qc;Nu~|!7H&^{Yo2oLsW?7Z_ zoaws9X|8Vl{pDo-)RZAzaRS$+EDD7(5qSt56M7L#J1>rT^bg*5BEKGE?g^>uU- z%7n3q7!?1eb-*>}*~OUX1}WFjBT}01da7!L#cjK-mDTfpP04gr1&PO{AV@l+K-A?h8s>qQ4w+8GGyb z!gZLIica7sIS^ZGb{kx$YMtW>3&)XLTHavbQNE&~<5ju(KJ#|`wN?cO4#IFN&2!gl z##wTEs%rVSM2N#weEO@U7(G+3-M@>>3K*R1b;5mzH~d~L_>8QPfNyldG_S10JDW_U#rdpBnxVMr_tc?SJ9ac<&23A!*!y&tHs;10*RY9=L7!e0Z;`GLg}L>o+X+X+Q;=-12@}LPj4J(&Tw*lU~_2H z?)?`yqBig;aG8eGMR#tgaVU5~<}axEts%|x5ESNgbj)y2;9OR}`St>%y^A@+7aBom z9qWLLG~cU!z3DGO?smM~C+v?C#kM*Ky1F}(1+GWEOC9W{e(t~j!xyh|T6}t;`VMdB z^}mr;Mn*Zr-$|b;eAD_&Z>7WCcOQ?Z3Q1TwSXnpc{r!uX9mcZ9AtBiE!_^`j%qmBd z`{H$NrRlTAj~uU1%+7k^aO_{S8{q7ig&TT!pfY%FrnK}4&b}TF%Y$Fw%#?Gewv>XR z>;y5>apw(NKxpSF=c9%1T!aQ`QJ7gYpUa!o%O}^21=cT`egh)#J2XF@h#)8a_mnf` zPiLUUy-=a$p_nIE5M6bc&KHpe8`FzNlvh9c?FD0Nmh z!c^7VN`=$(MsIM!a2*bxbd0g8k1(AE$D4nxsBnd9?9HNC9X@nb>*^Q$Hq5nt^BR&5QQ&H1tOn&5pQ5kkpTG{Ku z%u?~zHR${PSJ19coBc1Bsr6tg_+Z};_Gj~e9mDn?mUf=N?$tAK*J)d0_`lPczJ3Qf zNZ3rF(BKyv0wK^nV>OPPYWEXONnSi%?Tjvq^LR zxbC%o=l=csP=n848a7N@12%HCP>N;bJ&;7c!FX^6QtsQGGw`CIe4ay2F%XJGybe}W znfO3+3CEMp!nNt1Be-KhoB28~&Ewy{afidD9M|iU&E^`8l#x2RtL(RIFU4LaqIfu3 zW_WECpYGQ0#mryr_J{T%1wb$Rcj4`6HQ8as=|j^4=l@)@z1Al;$IMVIdXrT*CoKJI zt6rgkC4<|I%0nvWE=HS<$<*^n4d&77)$&LFiW8|0NFnL?7F}(9vkj=#C_APu6>~)G z^2V17Z$Hoxkq+Lmpm1ZUpZJkQRTM~{&&*6W?%qzvMqplCX+lq7=~8FUl}NJY#-^jK z{R1xK3GwH}RO32^qEXqEy2?3PtTI;Zf3&|>R=Uf{F~*jQ;2~LXw|vCpmiNtgPs8Ry zT!-jB>+JDvdiaS_FNcabJKpHA?qrVZ>0F}+>%-DA7g{<30__0oI~NpoS z$8<__qsz<%h8Csb1KyY;5ny)_vj1&q8)m6zibRfNB#!#wCP<`hVe$D1tKuk9SgV$t z_8BP=(=qYP%OE#g6!Nv&DmH3U`r*LLuy4FxABd3eF69!%%bcsfQnUB{+1^H=tyZP@ zjS%&D7X}gLW9LGm8kS;XjIPJ(;qjIu;aHq%C~bjX#4N=~Kelv}=`>qP?7Wi-5T0lJ zpj@@!$lwu0cfe}BqWpL4mgT6#oW>8fXenxcCoDfyIZS(8e3G%WTn<1&;i_Fg9yq)m zM?B~KJYkeo?ze{#n8?@0dDeZr@0P8MQJpu|7DBF(UI14(;&o9-&{SL3=6xPAiZiCXm%Mb^$Q+54Z*B;*O+|rx$*wFA_ zmFXkPPQD1ztHN*2J0Gcf=NgSnoP5Vq`lqXm>TwaT(r>maN@u;de!x)8)MT(XH`^~{ zS>|`SHm}Q-Wb%UA|8M7avOF%MYum>Cdx(CKtslkCKX{WzS@S4;cB;r8z#-k}saTuf8)Slftz}^8a}H&Uh-^H|}E}BOP0zIOee;GBZ0yHp$*(Wbb6l zIFY^gUK!bYWp7gUDl0-(R>X6k|L=L8H%Wbx7q|PquJ8O|S$mNJl)_rEb(hUb!&~50-~_nk^_gnfSeS+_T?`1I+?mU%QNI? z|GAI8sR}Y`SMMDusjiv4?ee!JC{e1Cy7TkWDF@$Y`FH5f=CB~-7A8`%@Q|o{Lm6&r zi`A+lq?#xKkFH{BSXn#a<)2?mJqT`gcJ9)$Ph(!fnn@o`GChgXE$uCal zS2Ue?teJTVGeK*$oTR8qQGrftln0$gc!HL~J6Hf5#ss(E?WPxAzZz8tgyPVdLZQ|# zJn|xDRG5_<2>Pk4dZ`1-oe(j%gR;&O_#GR+E&9=s5^qm`f$c#805^a-Tl!se2qsN% z4VK=ya|bQP0hy;7uM*Kq&Z;gC9|;I;-3D-u4>)E50&Q%Cl`X$M0`*ZQ_y_g%^^hv? z`>(1JzTh|?5x7ExewE~NxBr;}D-#{$20%1uAq5Bp6kPQ=0i&xR@NohX#o)G8Y^bCr-u^dP1N}`=+}!MN`1)0|-S~*tO#Mbnr3bCzu8!8AGp)upTcmvO zFm+{_y>aE^w?lGqIr%{uC|Zc8}Y>9jy2<-(C4+TBrd#O4W{pvoL5kv{ z!U|FNJys|ldB*V5h@bhcl6xyyJ(@aD8G#X&m;&tjC0NK-@5U3ayXs1YuWlbYsX(jo zp`i=8F-0RBZ-tmXf@sfPtZ0(l$zt-C1|bH@7?GqyE%cvgFeRP%5UiQ974o(k!?@>a zYABXqY)pBJU|&nd&ksfCogVyv=69b^ z&bQB^gttUZ`=7 zCB@O@oX3f+*{QgnE`aowSD>+zOW^mnQkSj(EJ)YLafbg)`9(RezvA60OB;`y z&Y9Xr5H&r$nh9}VIx9sZCOXju(SLu)*x4~fb+80GmY<&YyPu;^-p276-w+T-70BX`pbV_v#6NNh8gN2b{CycD19 ztNe4W_QB88an&`--WZ zB7=8FwcSKes6q-CB1ayv31gG6>J>CW+PJw5JBJ5o*zEtV`&!xe%6=iK@yVEO4^i-_ ztfU%a1aH6oQC~N+>*`V3eSD1fL^ShkSvVwv4kr$xIosiO>(tUIBeZ?H`+ulLO7WSi7*-6`;r58l|Rrtp0X+pc6c#nwMv^5Ee=ItM0i5J6bU^;>2^C(h_gXX zx(`ld)=hd*GDU!cltet18Ybl#VR3a`%h1%peR!8o;czpX=F;pKz%FR{_u?}7)_A7( zH+8v}+S6;x9|(rOk&7pJm%iS&CFLRR_?9Ng#YQmMa&+oCx2jWxqGcc3P5aDR5Ofpo~%VL`o z<-u%_=5>wK~6Een38YgZj$j{eOs;Eo!kOuEX-cFd(zlMjxw9T(s)C*vQ@ zx?_St9Ov>L`aC!1D3bc&y*ih=e1l9Wh5Cb~`n;FA=MG}&slMxs^axCHN-QXR>6=dH z+E=g0U9BgAGonb3H@PlMllS^6RHPJmpiuqv_)XqA4NL*qfffI{?{=K0@8r8=;h}|v z5y}MR2C!lMJwMGXMKX-)wB(p$D71-E@1s+IDOYkKkIW%!12X z#JYt4PYdAof+c?bx25ygli4M=ugar7bEaV+cE#Nm$%Ni%q=LADwzjqaA}xa%1_ncV zvxK)N%uU;Q5fA(>Oc#yC0cQsyr5v+;&ups40KVjSWdhb{&t-RQpn7SzxJXY+^E)jw zUIsLULMd1@JZAxhA}~4F2GDMB5=$#8>}M;NvF<)a3mG@07y2*20S1C!5dd#m6SwjTY*L|%WnvABV-aBwc>anLI%xp-vFKL z3ztSJAdviaDIxMP@;b;~L4np8w!#;cW%UH+ZXfpRh=mws6mTTWkOHHlriO2G&UvSn z<~A5eq2bu2yb2`jHCLuRX4Fy!ted}xY-D0~=Bh`2|2sRoUsIXxVExgGWR~GD;IkYy zaz(B7b>GjEbednv+*W|lVa&o|aO1>D2(yosj!BB;%{7w2%v82^V;@BjGYKH6`=~_gS?yiP&cw#anG6&KZ#KH9E!jOVhj2@{#nv^9 z;KK`2L4uV9Cx@aK~LOc`?A+%NJ8;}#hS5i_>M75h;^)GTnMCV|x6 zsi_=6|Jy71XZ)tvl%WAVB$`as3=ZYpgV~HE(iTf@YMPjk3xLH;i0qbwT5J2xz73#z zeY#iG1M)X>>N~(&_o-=(fFeGN`G4o_fE;)dJ%$m*ZU4>|x%`|31qJ;#ZQol1kdifv zI=HO^{IRq205#})NdYeSj`DIlU^)Zr^4}TiThO!#AaMcU`~o5!rF4P)F08vt3vwS) zA2ods0`jrrh~#5NE8{dT4jRi7SB z9IZbD>_mFoJF&C5hn9~KBo}QTJ+|^WK0qDFxz#$yjT_k)n?66{@*Ez~Z?x6UQfVC< zYjzaS9QXo)m@nohR3vr^@-+~%E8bSZS6B6(?eP@~+|iCWHg4X21`XCn#BZ)Ui|>it zReIwydP~!3IrcY4@VO*HMgbo(U%>sIoPg-Efsp{4#bgJoIz-E3j0?^=iitvoLt=D& zjGs=JTUNXj*=Lu@`nl;Myn4=X&0pOOl@o=VeK>N-N;m_UIrhooef0|Bq7S%{?(z`3E=}Vznhyfwsn8a5L zlhHv@Gs}j#?_i`e(ucFX&@7;nv#m%}REXD5u;KVDjjYhjS~n+7cqZV65|*v#!$wMp z>htt==N->|Cb*H@a8<+>q4{bAY%oh;@(1LtJ2*mi;IIKY55S9Px6LE}%aP!*=jcoA z;4wN9e8u~~&}04;NL&DhSwLf^`rf43R?cTkALFxTOoGxdr)7N@*%g3D&>rBD{3x&` zfN?I0cdIDfLFBLt3mCg3fQB4Ql6F`EKjT%!cV;RUK~u}uoj(94y8ZS-mDz5*VW4{8 z!BvFuwb@9GFx_Gi?n5=Np83|^`Ha7R`=iDG?k(63fGxcaT(inG!lL`%{%z-9k;PWE z-OLRag({!ZRR54Un4SJ~0?BC-s+~1zZGe)j`XnUQ1DtRbd~?4CB*XgvMOGnvBuDoQ8-H18_<3_df(9|7 zBC#xir4YjQk%+KP-VBC=L4+c6PYTohMy~QfCT7qm{iuh$&;Sd;h$|7iGr(hCMWW@e?$(3NayB+A*}wT^M*kd~h=P z>~5r|xE;a=M5qEMBRU`m$R7j&_QzIaRb~dvwVjt^jvsgzIg41O2(R}Y{ae2W5dwm+ z!$q-h63BI@_hxJV^uAlnc2!EghzHlO7obcWntrU<3D}Mn@x3*ey(jXg*QmV(v&+_B z_h@D0*4?B(Ic;-3^iCEkA%WcUm!__UhJp6%VSQ0FDpRx(pgv>(1s~X{qxm;7?EJg1 zsL{djT$0jdNaol+*SKLKD0yoy=EapCiHN_y%hu}6wQ6)4Q{APcTAl?YASPj|dNg?u zhW*23r6>|od*P>ITgcB~rWS>Sc4-xwQTt3#_t2-Pv@S3R_;IK6v23R!k!2R9xD`W> z<(o5pdSz8Xt5~!0>oHcTvEVsfX%1bGh%W_pkxeMxjtTL6Yq{pAar%M~=bwgu6YJQ^ zO01tP$kI~^DL9(;EVJH~A4piaV#NN+A1XfU-ZUc-h0 zEH=((2?pLlA#l3^Vz~YvWjx@?P9$!QqV0Jv!OZ>!Y#GeOPyUm`=m-p~u>vu|e@sp9sCG{MBV)tC@V2Wq@>9j3HY?RoBPyH!!Odi7BdN62Re$|# zls=DR^X_XWOntSSXQuDut(BFKn5f`j>JrvG_fO0L*dH9{s3G4J!nhfcOqAI%*unEW zwMT*SuO7=NRR8qupSWkfwU-9nW~qltyNdHH}9QnIl+qafu7CoB>Fi-2IJGU&&_cN;Y@{L{Fs0~ z=DSFB%NKeQBs$N$a|39NTz-<_n|+*pK%U zm?n=izhWNOCKVC&M2LH=hDZ2)Tb}kJ(N!M%>$y_e@*~t+yIg1gYHQ?t^3!VN{z4*F z*V^T&@o3rYais0+8`~uQpDq~+W#YbX3%(*|-CfdWQQvZ(^*ImRX<7?+{c0l!Clfo` z;`e{k^VgfoOA)O_Q0b8Q=AtnOhlcC?K&wN(RE#-u5ii7FDsyVVR zx-Zlptmv73EMZzyn~0;Rnr|LugO5hmD@t6M9a&}1R2Uyz3Tv<{v{zd^D}RrV5fEe7 z<@DVmsB8M2d>}E?xSu49Jy`0a8@<_3Ej}-C?|mP=Lpj)lY?L}v(pz<;iDDCufR{f^ z-T}!eW47)O#d;Fyl(0`*YGxLxdlnLRP{^L5Bb1VaMQEfT#pL>CG6)1|zrlgX%gcB0 zlBc^I-e5|v6)deykq)@ai&n;!BLt`8Da8@Lj7`1gr>WZL-!9+8DZa$_rxYq`tE8Wu?|i+@8ru>m{%qchdXbP< zByw9(tmU<${b6$C=%Y-eEv2CQN^`IC9{|yuZKWUhG3Ums6t}q9_|L!k2`<>rrJ6jC zD2JXqxa_aq7)Y4E@6?E+edkCc!6p1d!r$LsE^|5RVfgicKI^L2?w|fp7RwPu>mjy$-^f?97{J}&GR8n&qi z18x1mT)kn~cvL!ES+d(MSQiG9v7I*Pe0xgu@X_@~e%h01ac>1gAiWGU-^n=82VMgS zx^qXmY$4!~jf9-K6ap;>DU}n5o3jfKdf!?I*HxRtc4Q1zN7UTqnNZWwV{z#XAI(ta*{e zjuYM{-SA;@D>wEKh1gvY2R(#xf!JajO1>mvO2L-k+B$Azvp0*6$j(Onm;{32X^^*$ ziZ%L-oe$mMNs|+J&3)z9eg1D395=w$-T==eH*g1`sWop4*UILtG;}!%(0qo`c>uM{ zG@xlo|I?g$5P|3OdvfP>q3<>L^%W2!8`o5Gv2-4G6B^DgE3Df%ICNYekD%XNV8*{W zYwQFHWWh#^^b?@l*V58LpEuBUrz&>(4TxcqtArP+!9@7O?`mHVG;obsxkA2k9LTmb zPGq>qU+Mcd*k+8)SLHf@I~I@?J>N=I`30>6!^XfGvJOrbF66|>`0DuCh@h3-!}j_W z6&XbR?TxnsIc$NHg5uQx?bqAfFbA=%l$#y5!-Ekj|JcTjykD!TAARkrJJDRS`!!Fy z|E)o%@U4%lT;}(v^2yYzl>yJ?%fsQcG=5Q!`TK(sP?EvFo}7TCmV!&8Wo~h#RS7ur z)9s1XI-haE*IWAhwCX?E&2CrNmfPc+z*eb`eX_|D$f+2b=(BjiOFJ=Pv*}_T!`qJv zm{A#4CLwueUFaelohn0ZH$l!9!M4~^$OacrASe(WmBlt-Ro)N~l=9LI<2Jndv$s-j z`HZWD0OdP`fpKA1${1%*>9JGO7oo+eyQF4(AfzydqJ}bYm^Q!3En#B9v!is&(f^{5 z8`)i*g{aGwrsc(I2t=5Pm+!OZP-E(SJ~JEYf142?f^j5_dPaO4YS$xJTXl-S(F`Zo zxEqerg#@PBi`+34|0bLx043tX?9J3l@l*l9R)!8yugNlu7rc2kvh;0KM7=7${JkJVrM!aWe2^HRPe`vE6^V;dGR@v0tV5C zfVM!7jJ6Y*sgK$YyFs=jY7p$fJir@m5BnOQgY&y(w-S__++OLu)PmOl!RmPC?d}k~ z+xKmwh`r>%3;#I=m3d$=1!Y9W%_rlf^L7q=L;2z10{OC;HnnyuZ(W`^8bNA*4lPcU z|7x*)KUTN5>Yoje8~3k@))gY2zny+JOWX{p;ud~&XiA8o_4Wrm;LU=>qa*FNYs|Al zjl?)^vOC`ww6)8PYaQeam-ecH$Dc(%J^6{?)3DL*j-erhlVZ1SXyqp2$D-^~@Y$FO z`7z(aFoR5xwlIW=q25%7ed~jJa_-wpXD6>73QSj+TCQGngjJ@y7+~U>?1u7=;v!@E zEK8`(Kk!DyyU@*2K0p-Kd0;M^Q15lK*xeMB$m^%zkkqljF(n6_@iojcxDDUdRcvd{ zS1YPN|B|-gF8dS~9TFQ8ARl|-zN=k%efX}~R{N8H(YQL3-LxcIzs`v!%)G~|l&}}` zHT`xcrX~R+{@pN}zF0FEv$v^L_!*eXME!_#9~NhZV%RU_aXMM-oZCju->0%KdU5Yf zN8<7J23D|fLK|(ypzM1I;r_y~Gt|%YT39Ue9}DLd!&hXl;dx>SO%Ti+$(+?{TIJC| z98+=Uw%Ml{kLIyTUqr<}zn4(A9y5b!c2&V2JNMx=hN(JykZlwmGv?KxW~390S*e7e zxL0~Cr!Gl9SXR_^2ajlmm73WM@5vF&;xEMiR{TiD0#pqq8+zzId)9*}&Ts)-KagzL zdDb=oo)Z`;%_dI%p6!bQxgZb(ie4K2kKGV`pi%m*&gSbGIG1(;`uh5!z=57@^tM#S zIFVT#R8a8%Di%P-K7PTxTMdg2t>BIon-B&2%s?bf;~oCT0H%2ggkhk$J#NSXw0<<& zo4p02Apn-ofkDN8S^Eh;zjEf^jqNW~cE`p0BN|r+-~O&WbYCr0biQU*Os)i62W$Jy z-@UO9wbgU}w_+-y_Am(NZJqN`*lGHsxYWz-;rLJ6GQncr9{XSNNBU{pere9n&DCA~ z7L9!1YYKc(hJrfEG`#gietRufMz&e)3*VTihCH#CGseA34IlnnZ2EL1U9jIjRA){q zA|Y|^Y5PyM`Yo|^dJzX)LEYjNT%TpEcND3mO5z@93s){l!3jh{>=Ql+^%gF-{;v7C za#dMT>9E{jZNEJHDj@5C*OkRkx(!*9yDl&RpR0v?f6_f8K!$rH_P%)8*PWC2f|F$zwqhLjv1uK^G5?JJ0+`oPV zhsiQY!WzZ}&57VR$IP0+3ev|c;cQV32ACwugC8Y3f)u(BH`^Yw0d9B9KtMtsbN>#m zXn~JM3`~7R^_7e`sk>7jX{&@1tY-EfzsE=b0=e=TKlB&D00LeU$$J(KT3rJgy6@Rl0|C&xJNGg{b0319bhmM zerL6SU;S86U~$F8@{r$p5>z$eKIirNp9tn|)xZO9kgj?b&QS>oK!=&4rGb5q{G2DH z@Pwm!M$zD)qyN<+!111*jPilab9wEc0|YIA5NMF-Seut;1{Pqn{7)d|7}Rk30J4PI zyg^q^r~i`vX)tZPqAV%ww+|e2cdgDL(OP z3P9H%gF>3z=`bX9{31~d2wfxk>5gtCJw|hZa{f!=lNF2e>)3T;kLre|hQ%*4a{Qj!8|6Jfqn*b6U z-pgj@`bGyv5t#q7$leQY0Ac^NWpp5eRrysaKb#Q7Ll#%2?D}lETa}9YpPAG%`wCe} zQvuv(47iwV8rake74U8>C@=lvU57!BY=80=nIZ_ivkm?hA-kewU8>a6Avekdl-(V9@}dD~At}0S4{+RRBmPa08Z*N9}(|A=js${QvD5-|B%> z23_4|)1FW}0q+HyVnDgo*VMQ~bRGZ^-t|fLtq+jJqC?1lc2LA^yQ)rW9DE{ zx)^z(+J3Rww~>zY^4bL}6c!|H_n#Nv?Azu~Qn*iMQUS<(D%Ldmyi9W#bTv%3xC6P( zqC@+Hkon;d>%&^JzF(DwqJ78F!W_+_GpfEtM~SaiK3->f1|2{ir-(mwoy1dZ%dBwx ztM_QHdSHB$Zjm{7yfL79R`p_la6+i4*ZHd!$vf&Me;5fACph&l#vI-t575-H=aM`v73A% zt!doYQR#tIsaJZRW`w==$&-;Jp9ZJCxQS~!EPO^d^}87winEBt%HlQyW}^yw*Djd7 znXRnxL^+% z$p1LubUM;M?WNu#!_i|MEsN=JY}o`p5dhV^2DI3sJMX2gRp#j+V>DCW#{We9zE~v6 zpe}0yIFdU3#2|lw69imtk&%($+S{M0ps~jpL3@P#Zcef#&~=hZTUp+}?}1D&fhO0B z5#yW30Ea=>Z)<-7f(H9{DOuQDhxOq@f)9N~DCp`{`(&O0S>B1Kan?_uLtR2kOZy^{ zeP8ijvhX^6$FN^a!syCKeoZ;La`?ot%wqp|D~)2FSS%t+{O=^VrpHM|7-chj7HkG0 zRq(a$iSF(t{J9M8^yZj@vHx$3Ml9EU2k#H`AW<8?npZWQgQ0dy*RQ+xOFI%X5 z#@dvUu1tEvt9_tOy!W)#NlQUkrKP;wcWi;8!x0LV!q91@fFm?}iK5bZ=UJFEkfwtX zhFY~63+KM87tS4BGxECOjFhE>-F8?`?`e^<&U{fGzUvtGSNT5~zUZNu;Wntt%WF_c zw;Q+9>9{yaqZql~l9k4V@pE>Q7gb^hMNQTQqgV!gKKs0N&RCT)F_kv!Ns|51&6HMh zKx@Xl>Ws-;@PJ2kcmvBRufI@Ir>9$<-Q+Ebh=!>~i}8JGXGzTan4r%a`Kb&^3twvz zY0dBGayE2{N*tl?)G7bGtPSh#8a!fzjT!3jcqrMOXxXLngrp7jUs{Ams98ziYW_bh z0LQ*afkrR_ZZG#CD3~dS2ujF^6-aFk2?2O1XFFwfICRlS;C=Ei-n=*ziVfwI$7YzN zoP3$v5*|crO*wzWcL}nD(B37Wj^3*o;RgC9)&Fim`p+ICy0e~SF! z1lf1y6Iq7hS9Z>`m6-rKfk%)$xo+`?=F}LGrluq$ zc{zHE?q2~QKO++Pon0w0Ini@(vRZlf@vQPyhZPQAj-+IGxZdgZOmSOzJ8v&8zM%|4 zdO)zN{OR2oTDV~1cuZa!3|9&ug&6e7BllJ}TH9kW1vNDJ=$tV|yG=eoax|h2yoEod ze&LI&Flt|`ouC}A_{h$U7@w(FuJSuAemG`W>r=g|ldd7%Z9-I)8`A)-6VMv%wj=Z8 zEfU4~7mm(!C6bQ=F;KWe1wWMr% zL>f6h?hMk-MwpgvpV=H#@? zL?`x|tcFTkX?1M{->_itj237Xl|WJQ&Dh+Sx*td0;OcaRZdI{i%W)uix&xR(|LHuY z4JxS`C46>MQdon}wN9q=(szCMfgjqS(79QxN>Y3b>}2{Md^m@khvJ8w=u z-GbK`Sbcwju-ex^g#xmWhDS!S->kg{%P*7NxIjP9y8pQoKGaF7_ai`T*g@QJHgcY} zRcOH~+pzOUFamJPfNfreIjg_SO#a8xQ)&EjkvT=<_2GBM@bK`;_OtGqnwiT*`6F<$ zxQ$)^`PN7K{dlB0mvO+N5hEh9?7uSVFKAb?SZ&qf%i(6^{V(!lytzu=v)tPFGrXaE zIO+Lwn1^lhQC>l~JWSa+Iho+RgmoH;c!ZgCWyL3T1rprY*3oi# zK7c~QEd$AM)cH(VwUP2Buu{%`YD+9O%>JL2ny=kYYgEPm=}^^+pEbpc8y=cAI6r%~ z+|qCN&Mf~6iiBL!l!FjM1LrC+z{HI5wfcfIs;)2TL^b-J~f3Il&vghi*zq_9v8MbI5 zLH7bLP`PpG*ggRtz)%Myou_;At_yWGVYDuT0x4f)s0{f*`T@y3u$us}Q`MdS?XH4s zEUQWh$s!UDm;QDBt9uqU41zNKYQ-4W?)p_}hY?di4wwEZc2qC4nZ7&NoKsfs^lRN~ zT>X8u-zW+fa3PBjD!Kh4%|&*0sr7{dA3ppR;TJmPJ2gr|iRHsJKByQs}ya`eA;r%q6@(zIC&4_~OK z37^`Do0vC*u*x zK6Ms_%F^#?pKrI68gNr>e375~j2+9wJYK0b3FqPr2o;0$?A;rB@=d_!Y`)mnW1-^3 z&)?WA`kj@f4qSq>>;l+9q&sSm5UioNO{yS5eV9}b7AcIDeCH{$K+UXYRADFQn0AeE zskbvehmw72r#Izn+cB7211?qND7oEB)A4K==UJrEq$5QkV}#lpBHldK-q71%(P-Jk z=+B|ntTT`hB(2M{qnnGvjJaF37~khvdISSX@Vo>9i6?ui<~LmFLv&<2Y=ui}L_$&U zJD4&%O6;4Ya6S$h>PG5KrE_9 zMj0x_j(}UUCqtukAglEccyH|sGaC`(czQHF3KTxkDMJMvM_F`e3y(kiqtQ~`_sr8( zJ65A4Du1=c#X$QdatbP|n}b`~28U$=$W45vad7GYO_qP@QN>G3WtUi=0cXpd@0-~^ z9d4hgM6Z<^wWWV{Vc~Kb*N7-RC$YqmhDa6=p&((nX*uKXQJm{N-DyY!Tw3G)oM1n4 zic}U?v*1Plr$^k}0=lcyDzL8{L~P9lTA~OH42#9EkdntcYcsFx`Gu7ug(oJoX3G&= zTt@n}Dz>bd&Mh9hH{&xKB%pK}iY0G3ldGr}DC1s~Il^`d;f)OcL3g@39;0 zs?N*#TtBt~=pl1#!f(v=SS6BU>cS=N4Vl3$ivz6LMqQ;yIAl@9RD66Pb)dbX3X0*d z$37e{${G6$uZM$vfw;TdT{2lrTOEm zYS!*{uRjvq@O=T01i+M{xv;pnIJ^gMz5_SK%7cjirCR`-4`vrSEdWpjN-Xi$-75bx zk3*b#N#qQGITnOXV+DY!io5q7ZvX@Y4>EDG;&U`NEZ4}F8uW7VjbP4W!bl3Oa*{DoPhA~<3nye7v&7@ zo2|Tcl}z@#LMEz^l=Hs}E%y&-Rvf1PmQg;rXAH?vPS*mAYy^t+CVThhNXkK~UU!jR zdC2{Ed4PB4+3K6k(yhOtbbdvP+0|<+S1j6YJ7SJ2*8#Nn{PekpL7G01l!@;+3XwXJ zu)skr9t0dNlgkGsQ74BXWD4qC>f#j&dqRY2oP2vii;8e98STndp#eIQHkm3^;759D zN{hMBet3AneeYYnz0AM^F^^#jg_8jx`5~!AjbK!V)v+H3oZzb9-O+r<>((|_X&Ndb zyAMCTbR{cwU+l+mJAQrpO!@S9f{L>w4XYV9GN{|*7ZFo{mNQ07bV_v#Gl8ocrUUaH zCLV;6H;I#@`~j;5DWzVY4s~eCaE}W8BlGusqL=*l0=nWUj_J*g1ABOPB4aTFtX@Ux(?C!{<2gVo{j(+TyJ)m1_w z4`8mB|NPHZo{m?390oY%zVCSEM|Q^46w*+h(0d`2NMx#}WY@Vf1ba_bz2M;$#m_T$ z$b+5V6UFs7B{8CQ#{R=}y$oz4n=_ffcK!5G)T|jp(1} zs&SyGS1D(;b~J1hnkG4<2644ftJ-7^CZLfw5QUM=%*$&PUb$ z4UNdW18RA&vI<{r#DhDrjczFew7sBFm9N=1{CxgL5yn0nv2p~0R#&@0gxo^A<+Tl^ z>9rpFZ!M$D0m_kP`|01g`>sIq!0x-Fny@f|$|d-xjc+ejwaQ+KDqnpw#JXKKwup?4 zy|F&6Q3rLe*?vuhH)X5!5A_S5K0*E@v-q~n;Agk&N?mdYtvXw7XE>kXbsSHZ z<*^OB+(ni4efKB`AAF_P_U?P0pdDF86qKH@g)LZXSZ@T=gk;dlN{xX8DAJ~87Uu3# zE?R%X`zv@S+AbmMnopnWCn*h$$X^0SH0}ZcEvOZTdG^@v%;JC^oxT7hp0PA z7m4xu`S}$lzUE*j%pFhmxP)fc{K0&IhDh{!*E3?x0#w}l5~8O|`y?@V4~or=5!5=p z=k~8K7zBSLjt_gOR6Dl6jm&r+d(L8@`(~j>Ik+_W2XRFF!l{I{md-A1p@9~N8^-Wx zH*eAUbDsuUO4yPS6&vZS3W*`X7(^w!{fJbE#guF?r7t z78W2xFTqi2L}S07UYKmT0+6gAUgb6a{R8jSEA)98_w1;>urVw_AFuur1MdG-3Q>V1 zn8{WETYxH|18|xCT{-XY1p2!+Kl9CZ53j%DNnCvPXho;PtODkN&uV;WM((CI`=1@C z2}H*#fS5jXXd39%Sjv>i-7f-`3@1RlHSH9oHypKcwj6W}fN1sW?R-`LH8Nwbumf)D_bDbm*bM>M^a?c{Kg$`-9m4DUFN<=8>PxKa z(yl%82OsG=Er&*IjdU(&ojTSnH-7!RJ&kzecWJvTU`JH77TdU1{m7w7zPJ7f24aj7 zB1ysy-#wpt@Q~vsy*ZO!DN@vIXN zN=QfvBn9@KEsDAhV{?)ZniE5g7%+~yf2!U%W`zsUEU25wR6)1mi4D$>xCB5AEo5y~ z%Z?1UkTXGdFX&0&F#fdDdRy(p#DwP=cYEk7*G8yHSeeY26vWb)uzMi%#77w(`W(!o zC~W*0)+ekvG!gN0=9jNe3{*Xic^DXYia>fl=lzHA&x{O!w37UP|(8GLQzMFW_L@8_pBU67WtcqwKB;J!u zX<-sqVbU_6eBsRcii^e^skWP=QOZUKLts>QD@Zbu##ZyfF`QV#L*+tosFAR~WqvZ~O6cnl!us&S$K#YN)?W{x7QGd3($d_V?f3Yho_aP}M2W@_*Nw=Z3iw?Dr@2TFn_Zck}jf#a| z3zl0q>^4}DMmqgl$bHyuZG`w8@63ShV|%IGRCy9%OXYA`CPvlXw4Qx@A;0Z&J$36l z#GHNq!M)FRC#m)ymko#s^+(`cc7`re99*2^+){Acdk~aC1%>_y8!CuCDwuE-Q_>Cx zS*L|r;I_@)zqr3OI7lB+F&mcGWS%M`j~)3WASXe9lXLHFn!F8$G){>UtspP}i11Xs zCnc^-FNAjsMPUTUH^WeH2{pdZQE8;`Dz+3%y}46X-TTGuTgqZw@(`YJbdj~~Npe^ljn_Dw09(&bJ;udTDY`S{Xjx2-!Z z*(?ZeT;BHx^{F{zK=`$W8=aBBc7lxNR3r8J;6^!Ij!0Ho6KPXT6@bTOC8N_kF_w`p zfqnPa+u%u&v{*(OjP2JY!wi_ZBlJ3@T0I&%+{uZ=y|3&cFg4CF5lKlw3AUt2-Z77_ z%x^<7cd!2yi#J;Kl-<{i69%os zFv~{WS1^-t7;jV_2Ygf#y7qdcJv%7K`h&BocBO5a!9=2?EsG3lqiG0EI<9TVeUD1_66p z7iO7JrQdI0GQHI6M06w&CK4&l=O3l#(u97|2Lw=HVD!b2z6gK-OQN*NS*=QjRP3_z z6)}?wHk?Z3xd9}aSz5X}RX8s0L&&}3ij~Qij5v^#ttcLwnc;H_t0PKs^3}6Q!<==C z(BVrpw&$(FZu1=YPZ45Ovlmb9Vb9knm?@M-V|8p0woiacd4bZgyk3={(dVNLyw7eE^o_Lnl1Xg*{dtvuzq`8zidn_V*UGf>x9{e=dIIz zyM*8L0@uHZ-n+irVn_=!qjNJ?LgR4VlvkT^Uwor?y&3%a0=RxB9p$wLEuXI$`n%7% z>u*l?%Vf?IXHH)J)$TmJ&2aBu&^8`(d^tb%f?1PF8b-oooF5s0RCubwG<2`YiX9m| zM~#7r27;rX@_JL9QLtM`2lF`n~Ba&(wTIUMybJ*-`CNJinSSlsYMNqFddbAPaZ#(2`Y-ITBipzh)*o|iVJIku z^(2q@9XfEg09X|*uRuq9fi91M{PYtr?wB+#d!Tz>fB+2`>7sqlYP0|U&0}>63)L<2jz4$4>~p)qd~)a zXIAYB37N=h>+eNZkIX+ywTcFBdxW2eD`GIkxN9?y!ep@p--HmqXM~RiV|RRA=+TOi z3BZ;|nvr2ZVbC{(j8OifYCAQ6Tsi08rK zAczY^T$DlY!lZnh0}Z#GC6$EWNV3^9Yi_kz;A}`QV+mzYQ>LWgOj1bih|oWX&ky81 zdGYpIz5Kk|(n; zh!+=clFI1g@0l^f449e+|v?;8g zQLvGrR~o9LCnKQO$0ZRJqP0xeVKPmWOq*IMTQbmJGkTuPfd}bG7suFzAr9@WAOH5V zUIyubzynYCsLkmqWxQ&p$P+G>TQJ7HY!XJ##hTlvWd_`xeu6(GZq_Bx_yCxD(L_uH z)uR<+hw;I|!C1Kmm!+z{!_+4LIK2U%A<%OR_=4zK3m@}kT8Pk5uQ`iqRu^Z)0Uc^V z|F6V*g;Ee|^Rcn;xaJ?0xvTr!b}szrCcT8&{o*HpR^%2=IV zI6WJtS~yIt_pv^N#qpl5DG86Q-ZZqT6{=`Ja0q>*mv4ke@n1O%ky{qvmjyz8vRhxx>px#y03?fu*2UdtD+ zSsTo(P*OCA5opX3obtyWM<)99e}1HX7GCVO&CxANT_8wM?LuTlxMV5WzZ8g5&J=-A zNyD{@(GoM7qHRrA z6isb8VqX?BvSmlj6^(XZ)LQ{y@ebk>02@Xga3nlDJUnSWLw+1L%hq|Ffc>g4z2iU9 zk0TBCvu97tqSK(qyVc{X?uXUxrv(K-Q&$$8pV;UCwEO`uXe|NHrKi_7Kz!rG>f6!> zvQ_5iKz8H|TP(@7)%{nUT$-X;yHj3UWzbmAUbb-SDR8LK7KV7_?8e7CRBx|!)HIxw z{Itwu|J?D6ny_Um?sIa*5LQ=h;KT(>^PjlPR(!bdst*%N!*<>!j=TN8`{qYs)04?( zqVL$ON@R@!!>&>sQpVaqP@RV z99(ER%q9Fz1Esr~T%nBpg92Q#_=!|hERIFq?{+Kp(uD8+Mt#h)Zy|%0GfgqSD+L)e z>r~b0AK5k8RlD@8TfRdF35%mfHZCBF6{vT|3TQJ@&eJ|~w8Kd-G({vy2vj;2@Ah9WfPzv= zzlGTz^ah6wh;bi<@lg74(K{!B(~|`+I*z2J5#dh$u``L~{hnYj(I5FzU?cHvXEssz zCnwYmOT1V=2GItGhf;yEy~QCtaq(5!9+D;ktYOGjI{I&z2`Qkgp*saCpq_=KV2%Fq zm|+L$7>QXYJ^>OoX%Hl+groZ>+`_SZr&(frV)X6DDtjK1b+hf?d9*9dt6+<9Kor{G`X=14?i2((!2^G0ST{d&Z&fpX`u_k1^$S4K z_=Seb^>(y>xxq`eKlQcjLT3jdm8jd^NgZIShx8PhT=L1atl(tpQQi%dj`)Vk8m&nR zbyet!mbk8M-BCubPN#KDhBM)Ntp>5t-cAlfNS=`uQVdMB?raBnoyokQq@)bxa8g0> zAMs4@IwS3AQj28AwAk`vEJ;u!hf7P-7HbGF9(Yzx+hCF53oyvoBU5&VCW$*5p7(LK z$6c_`9eZ-E6XU}`k@-zJ(610(OiNkW#m|P9~NJe2poW z4Vc7ZYakz6D7r?=SoDwad}`^!aZ1Iq@-|_}8|hc+pZ!Cj^uGkY!p9f*?7|rS{hjq9 zO1+3iyKpl|p9yf~ec^7cl+9C0{$;pbc&TZaGLIJCGj4m!$kI4^Bg*=#Kr+2mJ{^gR zj>aS~KN6IP3=0o7;zi9~l_UzmB?)0R`xcJggXbYAUks4^NbIW`9>OC;xX6$R6VWhr zV}w8`YU#1H9;nI+6)fl)5}7ij$x(`6LE9g?x8zWmmD1^lxuxYF1;c~sEHz)YfJC=` zm;X~dO*k-mwxzG_HPy?YdmSzKvn>=l2=V}cm zq@I<%ECXXFWRP>fQQ`^PvwwTH)qdrlSpikV9JF7c??Q9@eMR+D+Y#mUNsu|F^QXw! z(mzV1s+*-7-tA`vsHcy2uf?toS9olvH5R>Jb4u;(B`5S?gfk`(5tJy)Zmdq}8$2(R zAnKtipG+69wefyIBIab>>5i;vbMI*Ue7<{OsSwD57(~IX z6N8I_OJjOCF};^2a&sO<*GF?ng@$BAKLv{FQPs=f*AdL-FL;w4ChjHp(q1_VJh}#l z@A>fWysL3FNUug?*p*KCwWis_K^yyKQt2Jn8?4@)DM{AClw!2+b;?Rc+wy~?G8hdz zF{oUADe4rr?gz|2EClZ`0#)Au? z0?{{8Wj7*PdvHLy($to*X4Rx=ipRc4x+xk_!}?=`2{=m`$-TWwtii0TvQ->|`beY6 z9U}h_$lzIy%5cVmT=%}18MX@O>zxa^>^ps_rD^4eCvh22VFCcsQ>kiRU>`PL zHgJM%pOj4k#&jV0%?~`N04%m4Ko^in)qWaZ5UzNN@jbD3r>E6H<@0t=z&yaAQdhm{ z4oEUU3o>3JKERv+a6W4|VMc(m_arm}`typziEi+ck!Oa0_2esnf3~Z`5A-12ZruM2 z!*!Xsxb$Fi+~qoM{8eb;DEPgxv#dga>{#4I?s&M(;e4~}e%OudP0f1m;+gg&aIv|) zoQ{6ABwUN$mhmc*?K?g#B88U15S52c%_L7i9*Hh0B^3=fUBcagM*G##NU|SrUta`< zd*y1n2M0Nvj`T6pTClD?s}xf?G*zEKs#@b^s*9ziy!4E~_^)Ez#Dokc>W07l zQEY9HX1yV=dc_2%MG`d_ev4E%NvB>IsK@*wKuMsTHEY7>c!>Z{KxsN1O$l#9K``5u zZu!QJVeeF}28vPN{0n-Tgwv@oNufU6i@}7MOI1v0a^RBCj@^++(~9I)mbcvtLPJP$ zQk?|guVSJ_F7sF7Bn4huI?}!5@j_=zO}94sO$`Ej);eLbB7T|91k>ijB0|tv6K#?- zU~?5oT3G_|Fbx2|0ZIpnBH+PYm{^P= z)Xt4JT5<5V{iQcGw6VMIl!A9iPugVxLZK)G8uy5ySV-+nrMp18A=L0{Vd z>ex2_25?pd{V94u0mO0!;(z?lK6LC3_y>ge7_g*pT|A;z^(W=LhlDm<0F;T-(bpQu zAhjoRd)&{Tj3pcYu8MluZRRJEHnE*XhL2Uf-pH6D3dYPwt6p5hS= zmo1t_fV$MV6hC~#S^Hr1xJR+*xnfYTU1eC={H1f!ff*}j(d$R@j(nS#Rf6T?DSbFC z3Bd?fh~Yp2*$*(0*%$~%8=@mDHt_DxuZC`u`#<Al!m_LyJ!Qmkq14dl{o>!E z38`Iqlw!g!2W=e8!9?n!5e4YztSu2+JfCZH|81TB3u(&X#|Vw;1qH)WYMX4PNx|r$ zcJhRY`TEr-9T#&UUz^kkLgI_mPWorz@2F^Lwek(MhCU-v6(=~JwL8oIoPs9eQlPvC zfr$IT@74O7Ufy_k;%g^_;kN6&%<3ax&9zn|q4wrN=tvG~q1feTki6(_TA626T8zClpPa2uts>K`; zVkKVP-jwQ$7$lr3vt7bPuLw#x* z;Pr6Y+P;62^FTi|U?T#EnAJ`C$9J2L|GcZ}?OIl!bUh$ugDJz|^FRBHaVEbD9Q3Q} z?gRiltT{QiZ3f=G1VkJFX8Q_Ijav=sPSY)Lkb>vO4#{+!kb@`3GlZ1Wr!YZ9S=-ft zjSk45(9nX5Jp%LE_RPzQb%NIfj%*3FkhfnUJvRZ?xyF7&nk{SBbIz12=BgN1!tReN z;TwC~Z_)v>Obfo1BNO; z5JHBP3J2%vnZt(GSBylwW*o_0)^Q~q<1mtDD{t>%eauN+FbGzn4P$0G=CL<+YSe!S zBLkC2P}@XAA&W1qESZu+QWbeHcg3~s@#$4_I06>+@J;kO$w3L^EGW233@-I;j2en` z(#8tYd>{qkAys0!49qwBDcT@c{xhyuLVa9dfgth-nGP74t`0VGy`UaD*$_tY6_ z2}sc(qm;EW2kCU zxb{3x0L!)bwA08I0T}j}Ea$G9etcEdKqSWh=)e&e510GHGZp2*Gu@YWijPqcXdM+m8>xb!fjA9zMUx^h ziKXSmX=SxB8e_lorJ<(=>&`|BV%x8RZ1fH4HKy zg}G@Rc#x^6SRoo1ud%LnQ^)R57YEW_4uQ&kLMnC*NkY&z6;uQWn^dB1vRfQEF#p@KL=|X#Ey?STtH*br z2BLa2Jbp2TiI`<9Bq&s&DU}LDDz2l*ec-CsD{dPLS?Pw&lyViY^BwarnVmq$A_fGVic10_7qLOqO4_GMlk_Wqb11u?TF3eif3KNs)<{ez>C9xZ><5=4%|q05q@dx z{!MiXV4k;GhB%N4bDz9-lY0y%_u@y9Ma&<_^;d#SciT2M{%t+aesl>=S!DU1*%*LDE0YaEu;ioIF9$QXnj$wa&VV?W&>;jIApL%67r%F= zvnl~x$)+D9q1BC942rh(2y-Q2?s}3CwA9+SRv%(XhzvlnR+QrTLoJ5FsC9UHihA%XJfZDq0k@(ckP9qoygCo1^ddVEB*4~FUb z?xHA3Du_fjQ8@*F!E+=r%738=m8Gg0y6G13Wq@K+G#r=X|N8?-O`1s>jD!Z+*-J`l z|76cItsLTWtJF2K`obqoKgoSkLAU2+jVlffK9x5mCnvbDKD;n2UmwGckSrf9xULR? z+q|$Z&QwC4#}p+f5}+-axa2A5;@2oa5WKLUU!9Oa7@41-L6pCKKzn=|BOk06h(Pdv zXe1)~5cl42_2U3}m7oLz7gs4k0flW0P$eq!)EZL%b_@VkwwrzpeR58(J(L>&9fUwI z7>Fak&&|!X2Rw;S1Idc2CmruU^CZVi&bM2@CXWG%J~`af$JYQTRj42=r~ULe0+x=P zd8zr=^w}3Dx!YjVip)8wd9THdW>(+Ucwb7}$F3%AksK1cvar5@k1}kzT7E||0jH;$ zB3$!xkvX|*YN8-_O6aC^zH0g7I-}_ z+Atd_H1R{0!AUu%O|xO0lO9OijG2dHC&4v#^Z9fnRBg}4bFR}nuLF}uo6?JX-E6i8 zJ$ge&-$pa$gr9fGY8*_AW2EY;CzSb)(@lv%qoKmUvn<(4>No3n8OBtGG3!}P z(8rk;9ixCF1050Rwl9?1?P2*uhOA&uP^LwO3kQyl$pwXTs_)`lH6fJk0flpZctf(T z?~2AhPUwjt)`cldB4y%`q^Np`fXhw_Mt}cCFGyjvRkI9ITEA(Wvxjsh}FL_Z5b(=9#rHsv@a)@v`D{&6*t&6&o)VA4*l^~2-}b3a?49T*T$j&n9rA96d^^{)Cnin68q zDGjW?2q%y}oJf`Wryca8U_mpnzw%e5{rK(6ep(^5$$}sMPAfW(cMc6c4za|sPE@KR ziTO7$mRI zMKJAzdoa7Xo<4Xw}wibNSw$n*MK z&q&4@$3}QY-&boKJu_NQmNX5+0uc?vsqBa2--AKt&D&Rf-vRpM( ztwNCDN*Ey@PMB!hVo;s-iYZotU?`^?KdY{mL*Z4d$3ex;7m8iMJ9B)U;;Cx8t<|Ebd@ zgCug|`bS87YgU$MPx!B8?dAcUU(hj2hnGhc$TAgjTu(E=zPp#YE1*s9N1xdLoXHg^(H>qfe>y z&0=$F-wkp|90CV?pdr7V_JlaJaTw= zNxsezH5p5I`FykbTAeH!=Qqqqfs1qYJqK~lk0_D&d%j=$*8|f*-p~jHt_&_Tgt3qc zTCJz9h9Rxlz7eyUqt6sm6jwL+quby(4kL^)4ozX0H_HP?n8I9#j6`e@O@&52glo`; zHO?DV=*dve`&K|GjvwQ%0=JyRmj4cOghn*Kcv2WoWHj^54ENU<;Rlp$xATD=fFW^v zxo9E=7@+~Ne=t7qPf<;}KTr(wL})n+>;8}a`!r>Wm$Cgx!S!k7?zw|XAN$|mAHG3@ zV?Q_Qydrr?CQD0ka!>qn=%1?BefoQu(!=W+gvt8qe)jYfYOBj^0& zzib}I`Vq;zHWt%n06Biusr7f-Bynb@=ztr0=$wuO%prj=6u>2&BG>=^J~wzM4xRaX zRJMtWmE-XI;zaJ3f)I)T#Ner(A3roDfxatU9I1%WDmO(IF*#Pc?1ugmM}Cy}KSD8d*7 z_M0b2JK(_s^*nL%fCz2H|JitMx$b6QM`13fDc?OrgMg;eP;sQ5hG*D3vPyNqt3H^I zxrb+@+K(&P&o+G**}Cs&?2p?%>cnGV$yQ$peG|jSeTH@|n$O)5UJ_Zo@o{UOt)GSD zuWCmrgxTVtc*S6G;s?UzXQIm_CH@4T5f%g1OA3Y z6f_{WM5w-VxAue;VBY`u=^sj-z?Tu3WLb9jwI&Af%i#sB>F=gxc(23O)%U4VSAMWM z0FV=2tu^!#H8(+1*>XOR( zAOTH~!Ad~L&pBXYImIa{6(@P)e91?J18?d?R9Ynd&;7bbyt|?W$tjw2_V9&zmYO8) z-SA-acXB{n2g%wt$Ny5R?fW{wFEGZvp~=m9_EUWDWaN$E+t2CFW;ms$fdAvWZxKb0z@s~` zu;Fjh_8U5^YF&t6q44QNvZ($?dN?I6IkBAjCk@ZXgRZNHRD93Ojqt*e`iYE8KLgi{VbtdVE1)`BDWg}QoE_AF41y~e zqPl472=qzl>S_t|bF&wC(KirUP0{v`FDTpME;Bj}xS%G^LOgJ(cgZmjh0Pim!HQ;j znrcj!%DKWvzx(1{rA%0f5a4P=MEc9#so^Pkx0Dbh!eYAhjVWraVvsa2F_zSdO2Xzv zpc!~pCUT3SgV@9Tgq>Sxzh`0Du-xF5&5^!-H|$faLOLrx+!v3Z(&!}}fxye=RKzJw z&TzU3VN>NXm+(CX(~`gS5?u3;0}1NaiZR2%Dw{JRTN_HS<&`-~^JY+}xC59V38z#k zi_>vIMS`amrY^0cpoRkqSJ>ir;#qYFPR6KnKY7jhh=Lmul5EEU0_D5ke|uib#Vi#i z=?JEp+N!L9qQ5RlVqaDHP^3#k`m>h2f_ty2u)a}rXw|4G+?8#J z1Zm7Tpmzpx@!6y0L1ZM0Ku{6hVQ-t^~*9lRSlDE0FddwBdVu|*WxGbZ@m9N zOgHg1KgV-$1T4j*4X-8Lc9VSTd{;Gjs_y&xG0S?34V=E19Uw*fgH_I_3$T$?*IZ0m zb(VcjMcngC4;2C`_8N2XZzoJytzz|B2CW{p+7C$@E?=?9RDL@VIjCA05kj3^nF&GZ zKJSe_mUWx?^J!QAa>Qi4+vlNF(fKkrqUFW?RZ(v1-n6p)f^(53S$xMBh07T2LH~!p z%Di0oaCqbr!AO$mi*;HRPTO9pOacg!0hoYFMq(xb8J)VHm$Yv}HYX!eJc0ej9STEa zO(ZVKDpF%aectu_2T4yiYSAfw9o4I`)V^qp3P{7oB^s)$?jSuXB6c*0Je=4FZvEfI&CRC_FY+TkOFSQ7gcJ|$t%*2Yl}hpE8L~+ zqb+kqeMht(aOvi6v`|G=HslbxMnt(f;? zvb?#sZ*817M!x57`b9OB8N%k!Vu?V%c7dmy!Uxem1wf_j@so5+#OghpwZVPTm0UWo z1BlB21X}<~NA&5xUV#5I3Vm8Vd)0jsatl~98>H`h1GU;eCauDn9O_IBI-o)}Uf z)7`b%D&QkieRVFEm)m?-#ne!Gt@H~i7#8MJ(mIj9o@NBg`$opvW?iYvk##&;?j}J~afP=spI9>(So^4)b{j@3i z0u@Rt*w(GiO-%lk%@KS2R{(}t4jkQF(y)Y#uOx6MVz~k{d@8&o7CP7dH_{sA98Jg| zL@;&S_Lp_1NOn97q)@m9WX-9C%FbcwH?v!eJe6riblWMmzEwErOw<#VIXI)$F!Tc2p3# zB6VpqrWAQQGx^j62y%K(yM`cIpR4n;WKN)pT$6c~|J6Yc=g_!lIr7p;gZSXo?qo+gV;na9Uc$G5>m{9u~`;H-M$sIkFV2Ir2%G&&s12FSW<*t zWqZe!TOOy?ofoZt>1eJ_8FKmfvS+{RAS8m@)*Y~GP9K*6cIL=Ke3pvvnpoq%-T5P9 zV{FibFFr`gLD}$&fKYBhn8m+wkrS4LU@&p8A*}D?8;^jbQx?8KQ^C39@5b!W7iiHi z8<3e*+_q=g+iE%pT)DWKr%&nu3F%0VEX%~WPRqV;yeFlRv~H}M5e~w8zmXdWpuPM` z){0Vi(@1o~Wr!s>3?yW+sXULPw8@N(q42t)S!8=S3sI;*l_Ao^>pXZ8=h#P+e7z2D(NYR)Eg=IZ z0m=Dzm`H^f?yZKL<{?y!5#p2n(l9ONDKjS4j~r91knvqbI&pK06r8HshEG)->T_{h zU)F8;GACp!W2HlI;1)TKJEKP;)g&6pOs#nQyhl6ES%#e%FZf<>sq7euj0=gb%CqQH zJ+ZByQh!ed3&6#G>buxJxXw2fOUCzP#sKu`Dvd)RxNtviEviY7{QQ3u$OsHa8VZ;G zkW-&;=+@b1aZeeTDnrgTv$ytHT}6>40_4wG^6VIRZV=}oSvu8s$M>toWZzs48->`! zI)5!VuQ#rC`OVS)nM`kp_A@NVttrjjxQO#PTWit6@p`sU4gt|1Nks=;|5bhb)^z!! zEfO1)p621CQb4gjr65@4&r56anBt_d-jy!km2*8(cuW5A9G$AB(Lsyc2Jj|edtH4|YO zyw@%M1>#cb?Jpm2wH}E6)_s0|KDl2@q>j*`ayCYwF^NUUpzSO%P;O?QvZ9WVuu$@j z3$W4X5tXDgZGB-gdY$c>+&C!`+)&e9+uzY3;~5C)H23O zBlL@K=c-=QiAkX{5*=ymdH$+qCz6C-RUf{>**!s4Jk`b(w}cbE*TAS|m6}zBG?y2%Cia)cNK;P}QzKGp~W0 zm#WZlBhVHS2oM>zPui{Zl>k2c-4lfd$lkLB783mbb%U9z_yd@5JT%U#c#j$2s0QVA zY`19-I`mOK!(IBTXun;-`%B9Z3<@UI5nJ{;(c(EsWH>SQf5Z>k%Bc3exEXr>Imq_1 z|3EX|@3DPbMc8G(|p$X``qupOD8m~l-aOy zx|N*9vg3a2c%2+@B`|{LG4SYleue_YtX$90&<)F;6@Y|(ex||{Cr(53-zt#0*uEf& zEKEaV(IzEbAzrmA;+WIKMB#M?fEHTvgJl*y`bTJ-h#oA2-Szr)JrPLD!J#KH9~;SG zFzsuUxz?IfJXQ^b_gU+}aRIndhYrik+OpJ19?!!iM?Q%kDXwlihk_bXcd$W$t~tv< zp&G`q>K(jD%4N|(+s|4#otiizn^8Zc*%;+EG$UKQGtcj1V*QE9Gu`gR5bnGA^Emr# zuHf(?L$4t75nA^ zwXaRx!sAy~$G>7ZR*(-PyM0`n;Q+-rQ2OpKn0)s0xin*-Y&a72`_*`)RJ?YASI5EK zww3Yr@hqsT-E23r{9=%y;N;0zi;xaYYUraUtp|CdFS*vE7801E{`Le!!2$bC^2CL(It6rN$hg(jTfJ~AdopEV> zO@Vo+1y{g98A7SU8rp?27E4GsFdJx&ia(2VKKWYz@^}IVv?=#86l#>^`))u}fId!M z@!NqUk&xH~zR!%Vnqq%J4KIC6ZD8^0b2}=XYNZG%`xCZT8GO1z`~3T z?oE*CAlJx}my9i$VvNFraA4S!tevH>q}CeZG&T>mN$_|@t_iEOucLq>iV-O|Md>9D zhq064k@#@@5XlH|@Y}jNj&WgbVTUb#-o!9lHw}{r5Ju#T%5{Id!ej2>XIhFEWPOr~ zlf$YlbZMjw>$26m-M90fffm+Kph}s-V@@3?SKR~Je-yY%Aw8w@N4KRaHP2%2Pd0J{ZJ=15WR-|Vi_z6`$9NQ#FcH@WAj!qL@hj>N%AK2;O*g7SmWg zKf2wbNIbbSnm9CRrKDd$;+fI=4n6vZr$HPdby)L_O>X+r$eJ)*u6Ybpj!`NG~b;_Nu8E;+~ z5xt9PXdwimbE3dJ{)_qf=cmRrAh!7=NdlBKAHcQo#E1MlZTw#usqtmw*efnWcI%V1 z<%6(3&p(M*ryph6_CNfT_|f8EjUM9~a9nAh8)SZVbRWYX^?~o1$nBV)FN1Qq@~@Zk z6%AKQmvurXBIhaFPTju$qE$s!Z9P1*FYK}6Up=gEqjv26)>m6M`sPa=Sig1^60k=y!sVO!z#r4v_Au-}q+#aJb?5wEzxFAz<4?*=gi4u_RmjUGnB1RZDl8R5_=V1H&Yua%uc~*uogm}%$>kq};Bw-S2Rx|Q+5jI=$ z_Bz4E+T=sM4i1>CmE!inYKm0NgnE$?U{<69aoq&m9yExZW29~}BeIz($nZcU*n@zU z2$ur7TdPyaRGp$RNGd6*!C2K4S2#>;@JS0atRGkSW%lIHr803%{5};qISH*y@cCoI zbSb;2qvq4@gZkEEvDBoe8aI=>mBkIf1dZyGE8acaZPcS$h-}rZE16$j^%yS^a{x1*IJ-g;|dHAMA_x_LlLU+Ax z1_&!hPL+|!?nssY!InX6_iysU8B)Q;zKv+Zhw?9@LUdA8eGA_6gC+f1-YJ}Vc#Gzv(kSMo z(lI_47+C#wlc@;H{B&14&^*Bin6Xf4A@;Ow@2V4XRhuR;9!y?T6p9YtX?B%{aY!=X z+sqaiNT&p+_b3%gb5P?(vv~PiuRdI_HE{-s6;jzD)FTJ9K(Pr4T5x>20BmZ!-341o z)-eqpk}54Nu(-BzHJ65&O7RH#sC{}0`MWjWpti^oOG3K_*H8Atc=F7#3q>*rqjBtU zytwK&BnzedWrD^2hEhDTinOow3z~^B4Fqim1@yeKjsK2`PUyV62}_HbSUD=d)K|pP zJp>ToDq{1cGDw9L2bSyMOSq}wRkGs9=>qXFh>cIN6?Qy>Y6xj@sTe96W=zXZ#h*yk zQyG6zNOOsEfcKhyD;9@;@8pbTgp@SUNkUQOcgx9D2a{VB^m9z#BPIo6aTBEH?|S`A z7Mob2h>J3q?QFLC&+hLxrfr}V$bP3C79Sa@!AP8hH1wXvJnRjF37_+uJhXg91mh_o zc&jVt^}_3E(06IHVh`v1ci(3B{zQ*F#r}f6Bfa1!20q_yTpI&v_D|{e_<4VyxXb+h zMbG|c(IBDRxBq5*B<7|3_&MO11+Wvj$ztS49RZxOD?g(rG5!o4L`hEo& z6Ww3axGTd>6Dh9Bh7wsv@kJ$$r+qs8#C=XPe9spd-;L?O(tEv47w6e=G<$R zXs@-v+1wnVtwUD{BGw8f7>d$^{CY;2yzzd<@-jAq2`uvWg5ct-^*9=??dvq^V97Xx+?RTHHl&i7p8dS= zA=X45bCi|j!|^XrB#;&4DB1Ub2v!u;G$ML)qIV>4W%s$Q=43g)M=c3v15~RL_Cs;8 zsY{UkB&Eca<}VBF`X@gU(8ASz$jO&V07P*MZ7?o7jSE(A}UDXxJXRIwQl3Ah1C76BiUf3h0mX*#CwUQjFd_0By|n5^Rf&~ z>S=z`%bG@UWP-VbOri=ej3uR*_{-(GXbWMrVhYc2^tqKOXTWi7n_upEmd zoF)&}(wZn8t9*3ZK1y@nT{*M|_7rD`&>7meVuI`4pUjEd7^2|pkgN%65%V4;LYTY& zhA&i0)9O5T7KI3$JVh4;-XA^On#{S@xx~Pa7T4U?%yr$NCcE#vei`zq!ZVEps?pTi zdZ`{tQ&wsR5iihd4I0`&_1tLh@Oq}`p0A!MABsqNEBx{x`~$A68g&PA)HjBG7m zW6B(wtCh-?&Rb|!GT9jDjO{jVi23dhkF`u*kb;ZdmTf6al%`R*Fy${)IxJR z*Y}x_iPFzC?`+@EN?Rn5boPErA!Bh(UCww~!Pgp2)}zJqU(B89x&kqf)2hU;qJ&lU!!NJ(r zHCQ~R4!<>@1R}`1EPzV~HMitVwlh0Ls$_|tAH}Pv&(PD-;*pX>QsW>%hHD}5z@s=y zmAUCOFs6l0mrK47V{_ojQ}<{8$tI4<$d!%ksECpUsi~n%a%23Al-JMWEhKKVCpi_Y zZDWcswq9B}cRl9wu*O#<>q;i8!T(&$E+k$NGTM;#hvr%72RwXBd(~36z@`|bbL-t^ zdS&|d4kE6Z@=Eq^Z;&+)ha{=E;Cmqu4iaioL1`ojS}TZ@`5S_gS|siq$&~l~SUsr& zX!O{MRY=ToR9x7WT5(Iyb?2Egq3qJb<8kXYzqBV>dkN|A1ovQ5bJ7G@QZRPl&8A#5 zSZdpGT~rSZhCz79rFsh42?==`Qd3ijcc-LqK4m|TaMgkrA~-ql-os##IoBH&XempQ zT%cm(TaP_^d$+G&$I*Nfw-HMX(~e%D#RN%PJBC)d-ZRz6$-}nV?`uLU73!a;TL#Z6be;3J=|{$w1nY zN3D4~o6IwBBLZ>8II$C;29>bDwo>aHJU_PJ>RecBGu*rSrMT#8JCsF zV}VJ~lc4DO*fC8Kyw!`gyw2}M1m;8`u^A_&76$r<#f$6jz=3!+;OkPQ()PJ^!R%b_Ck#6u(Id`atOR0EOylJXTnKeRrohm66<#*W zT6oqJLfUhB2=65?2gpo-WJ0ehqf(m#@(yjXj?G`4m)hJe)7UX!`AbdI&fJ%uGcR3>bVP`)?v{fImT5-R&Q6C~ZIO1Bgb= zYxw=d2d6c2s$gFHfgPnEbCuN`C}HvhUJl)EX5Zp=BaB?ydPxQHikQkns9`_xJSWc! zw7S>z5LmcV9QcVLxE$u3QY5-pL=|>U1ClCtjvUC+r#bSCH88tP;tbD~1O(%2&)Xdm zTq};jUn=m>rWkPpT4a#bL4W?s2Ht$iI}6i?s~WxEs^+?!AzUodszccVlSvs4M6lf*0;DjFb3dZOH5IoWjF$#dyz~U35_3mAaQAx|scWs0- zVFJ>Ua42K6IVT?&^3?|Gy$uR9hCx+jtja0wRYcDiopBXiG|ntM2K< znL)6@C8SNJJO~<0IS?ki+#m}^)Lzqn55_k2vGwusdHdE!K$!p^*b?h5;?f7Vr){`h zYe4A(XdViK>oXoGd|WhCQ%VF^&oT9b7P|bWx?cdjz0U)GJU0d+zZ(FF92g(?b^R$} zqi}bKt1t$b7HolM(H}p447wi*@&_oHNLgG@FPWzm@>|+m@6Bi9o6*zppkL33{<**4 zMFsJ0z3=-o#o+DSDY2L*B^6eqkP5T{?lp0vOcZqyiGPE|4NzIS!E<=$$n&mM^dl`d77HAgcw-QhI7| z?GO6%{YqZ_*zsSPGEa-Bo^qz8uHwF!$pT9*E)X_{omPs^YEP{@9nDNJfwraVWJcyS4sPdl;bAt*SiNlpv1I8UY`N8y z8$+AIBAB+3S`mM@)7mqsQ2Xfr1b^L#pfH&O5#O-lYrTecOX5PvryKDN3y!(BT$G`nICbTO? zEw6Gs&P$W;>u~rff9J@>^PQv~Yp-!!(-I%>G2m1s_RNM9~|MdchOP+Qj%124zNbN~> z@)_;G4a{*UahG+CM`qEsvGebb49+qS?F0RvXujlGUmfb(-lfgCZv+MAp&U-Gt<3Gb z8h`F5vMYc}y7*4j=GpY}+@@O=G8Cy!(2f*@n?=WvqKSkoZO)^KM9V3yF5@b%GQDd< zRksvQY_6BK(U93sADkIgd%fMNz`-oHo#^Jo{VK;APA};>DPU~{H@7N%FRzpmnta$+ zbCiiD(=TA6t*xO5o^k(wOr3W))$#xL&*9K9;v6ePIQB?5M)o>p*1<6|vS-6cG7~yB z*;}?_6SDVS$&8S!GK-LXfBf$6?|a|=;=2676qxKE{Dv};zbn~ zQp3gfM#$IeZ zyV__350{u!In``&;(&>>*(a`tAF?*BcFYUlH< z>(utWfZfXhes2d+d+D)Ooot`OJ10_4ia!39YhAS{VoxZd(w1Ig6fc`F#644<z|9(9_L>{q@?Bb`J9lXGCp|Ov#N*f%NO1ZWR=S03WBV zKFWC$079kK*^!1X<5JHqwxVzF)nau&gNlY~Xuc)1BbK<)9rOSc=rkwU?(9E)Ihn{_ zqrY+6iiabV*u_#I5CRFc$_d0vVv}(di1r9$AB(f*X|eCOgDGGUXc2XN4BWN38Lb57 zD!4>gna!rss_Wg71FXA){SJ6Th))J)hJK5ow<}CQXdtP|4ad;z>%BFH6SRF z{Z|JY43vula6v?G@ft(*qbV~ND~GhMLmPeO;!stP(^dYmy(*Q2o=PpDaGx)L-g;2F$xx~E2He91q=Ds@_W?! zcf5CAuNh{4cx0nV_NvS8Vbw^Q{DSlJBfi%O9(^~~pn$Pn`zqu99023Cmp%U~d(kbM z0nFUF&dr)%W&#$tH#r0L#}8nP-gp%NWJt2*ng7nbwAs>1o}TwFT9}>M`~KtWM-CMo z)+>4~p7#*^)JRt91$|dqbzcPUG<*Lz-USt1&emq@D<0oEZkhx?*WBTS=wKXD34V$= zE%4c^fOX)3fvF&BwM-REy1`_@`nq)U%gPGA@lnlL8m`;&3{I09Rzc zyR9*T>rUhbh!%`tBlzr*xRv|F(eXoxA(75)Dml679yw5mygXl+1w;q*1`P(??(7g2 z$t*2NEyTI?@3gWa(5_aI2s9gNqV9}&TcQ9oTIQH$oO>GtuA;M?S=4b>b6(V*A^-g0 zs<6Gl=gZ|K`_H}Vy0nrjoabwkc5`u?J2QrNv< znbmj#=?*9aBuFU0`Vza=qNu=D|E)l4=dn0X=7WqwB{YZ`7Bj1t50#6GKj-Q&cgESafa=m!?MEl!qPnuf~4gg80ChU|UN_ zELR3I%YLoTN9Y%N^TKsC-ETr`qRzQ>jY`HWwzWU1Gi)c556irTe#o>n*2#||UG;}= z9jRv59bpzrI99*N^k^@KB<`5Z`G~(Xp26y>=&`<3gXZO$6(SMTy#wOoV=UDt1hscZ zqx_7M){FU+$+&m7tg)I{;u0ES>}L(Q;4f!i<0`KdkB4K84DNYjjep(j_=juwhCR1u zfq(oK!{Jq3jAA4YZbDp0xSup-*90xn2Xzt*<$z!m*W82;mRRe%iiJHHTN5jZ6j9En zm8p~E_U6N_FQhx8%P}X+%$KGvV|$CuH-Enyj2Yr!0yrnAygsW02-3gbZsRxMaCh4O z^#aa?j+4Rwz{9j1$OTz`$MM^hxBvT(ev9hm@&rTkWxinKy)@m<3CBl8iwES2(^LMT zOw`)%MlaAcr`UO zUxdLRGT94mTXDA$LE#ZL1W1%e-Kd*`D+uvCox^%Cq`Prv&_uyrnZ1NPT$x%MG036aC4F)Ioj5`P^e5Mm7h{pVyR-aEngl*8P%l&Okx) za}hjFppKoE6O>Egn53P`PPkQ1&!m$HCNW%yGO#2Zch)7oiRCKwfu%Nb5wGPt%iHOG z7QR2S(-MLBH4?@h!l0=*x)cH%{z93Di|i}F_0fi@q7)*B+K`1#^*V2ah(83-ys-=QUugDJ-I9Slf51ft^O`^DbhOI7V z{lwJP!%!p-ejF@pKC47Vi1F<}g5DdKC~w!k;lID?raolFBvnv+^ZJ@&R;OiIH|{?3 zuzjj(y!d-t2I*g+be}Sv?C76C)$kdPtCx^qTQ?Y(IW%?+t@h;yBkKmvM~^y8P(AX+ zOL!0+JsgqawM3moSHxQi%p>TL>~5cZW;O4?UlvMPyh7@s;c6|B4puAr`lU{359#+w z@FQ`%A4?xRj_lNc%ICs9q~Xb-AHGZ&V0>A@r(!FvZdb83t^Q`HJh|Mpacg@XMr0>b zhe~F_E8r^p8f^NE;sE!no6ERp^4Ei_9}lj6##}i77?vUKB8JD=!??4v*>)M~`1UPK z`R<@n~+u*+-IBakILt0nv!kaanaU2_SZ$G9Ui&Av$_3v9Qzv$jzZIGf< zp4UGbYX+i&^=!#rFy&%3FbWn3*6Od3Gp&`*6BQo5AKCeM%YI%A(+`3h6qTlvOP2j+ z1qK=O8kvfS-ytz9De6#76^ndVGv4kAeX0CHr$P=(g|J5gtO_8-A3ZPDuAT`fid0e2 zYlP<%TUjPZ8*!He=d_!O)>hPqLt9UREqNw?eRNEVwpRC31O*N-y1t3cl)<7<@otaL zzL@a8ozYhq72R^Z+T&B7Hwofc{*WfoWf~U6A*m(SAYMQ#IBatdcf$3x?o6NL@7q*x z2dQiJLCxcsL{)y%yJ3sesiPhUyYpC?U!7&-TY`aR-{Vq^sXh;sEDlwlkbz-*sbN!m zsq-1`sk;i#v%&_G38EOpsp6V&!YCotzH+veUqX=zv9uwv@6W?@bEE9ER4~}DARNwV ztI#oxR*^oeSXfgJlZfm2tWi{o5WJl9Sq`?7;tmtayraMuUc*JK_%}q`j^E{u8Fg;) zMoCt+rK_ny464@rH+<}%s(RVk4#_n0ww))Q`KbJbaCIm!Tpx8&^KV~ z_;N(B|FQRaV*KEmHQ*Cq53JO?$s}F=8QlACHU4;o0jS)@|1RNo3|YPURMgv7*l@>P z*!Ueex@INz>q;T(JHGVG$n<~5lNPUloB#YHkF)aa>t7Y7?e4jfd(XXSyWh~(d4qxx zcPXmrU{AiNc+9`A_vZ|%p(|YELZorqT0`+HCm~k+!_{D%5|V{*ldI>ogqg9qxqg@@ zhK08NMKyN;tpHxQ!(J7k0OTl=L$Hs~t>on<5uO0+2unXCaL6W7JqH`y!JT$%N~447 z{`5Oa-G%GS`~1;nD}c!{k)|I0#^Wb-Q?0h*@pHOIly$)bh!@5Q?(yRJEzh)rwe}Wt zLuTR=@2OAGa@5!{iE@UenDkG|OdGnk(;PFYvs)QUPc^=J-KzRK$0c*;;Fs;+vXuka zN+%V%J(6MeVL%Ok;Sr5(8~H2)O!9{&`^;@|k1WQbVh2yv1RCl=ABzNAbTo_(42y!b zJh=EI*N+vaPRBAd*YH^y;FJ(JCqZR+odz2VoTWHbZAghfJb&6LO?ldi)Tu;&358NJ zG%y(9>j7DQ0DcivBc@-Q&vpA-Xf@moM=d@%e#yLWBeZ_oXm z)xm%3FHR8Fi0TUX-d#YCtWEhuZ}8-)iSH-QQaW0hXC5=kCfBjra~?LQ86@NjRbR;| zo0$ZcYOb)WG5$i4!lHk&7TOy&+s=*u1pV>0eh3@v!CxWY5kKuM8wIO%b_GI;|0{d7U<62M{=mx3tIp}Ju!1F?g0+AWp`yONFn&i4Ua)C*xMB9uqf+Y{yuuF7;du`H9g;7)(D$BU zk4_ntoy9+v#UQs$rYV41kg-Snjcb5fZy8_~0Q9XF0tUta0eu^=hhDCCy^Fl=dv?)x zW1ad|%T)M(Q~2}GZ&J0y*Mo=oeTI?Q-a`O)77sM+)783c?6+o^iI#D#psPG}w~ZLp zrpxy3d@>vGx?iB--2G(iFB@G6xQxGiWVzkPar}onxzF+IG}D=B{9QKkz+P4HGSyuU z#xF8Vu;%$(1A+HY5_L4QW#9l*wP#RSkqor@(Zs~uc&^pn5L3Rh*zD}>-6@WOsB^cZ zCFs#`ual4E7+fVx{YYm?+z^pP8S&D*_B53 zB;KcO#_1)n(AGHr>*<{C9SqG8GBc(ja#FX8j78}lEIqUsw$y*&xhYyj zCHOS$VKoKIM=J*xebFMp;#*v>;z#(kMAqe71Ng3pO=^*-u&~B>D@BMv1HJow! zFNcJ+!J~ZaRmWEzUx4kX(YprZhv8?9>q&3pMo)VG#s*ly>XIiU0mAz&|$*8wCf7zWKh=@R^YJ@X!6b*Oj++O@95_>j_%C^wKok zSQTi!)phoCuI105z?ZwS0e&uNL{VdRFF%7{y*>}HR#ffLV&5R!&{xo$+6ew=dYYgS zIdzP%C8uB(6wCoTVJ8a`7kxIEj2@e37(C+08Cpt)TJm^{IU_lE*YWW|@);Gtv+y~; z*t>;WmsYMgNwvcE^r%T&i>r$tl7_=4V5GM*0>Siw#S~~4uZhBP;l8FwYvuf%*YZ?E zq4~J-9Ci{{sNnALgh0rx5_M|ChE)k$ot(D+~-Zdvr7t;Y9LE;K&1inL~@{3>xc1m^>}rM8A@wf*it-s`nKFie?vN zo$u42{O%xhJcgN^S5?Y$m*IXY)J&yh&8R?mmwT83H1`79ImH$Keok24npU$9#bC|V za)&xjRlyVUMx0+RS3M!nn6LoELO7w(TLP^o&NK&&MQZ6Kz8tGfe%7}wKNCmCY(i!h zv_Z(aKI^h`AZv^1jP8RvEH&y1l;*c_dh*Wc=^fz$K! z%8UI>r*4M*eT&-7wSo8;$v=@r%O0b*51aivBG$I2=|NJTYL$yFWlOV}pqaNa;ro|F3q*gt~T48@<{p5qNvc=z^_w3hPJwfw#GtQNzn2v%r;EAlfs^u65Ao> zR*Kt1BO_SS{HF8h{hg)8R+B6*o6s_znUL==7>t`N*KOaltRMS!l*}_#}9^X?s`TBB-HON3H2q^4xZfsajq2tz^ z1J8q#r3ZXt%?5eJ==Z#%=R)D24hN|j+EEux&PWk2F17ms$Wr?1Ro8*V7vvZRO2EP7(3+>k{Aq54SQk&wY4Ns+i;>tkZ z=bbD4dU?&O0bK!Les(>WeK8nwITv%=qV>Pce$4sCSYvX_S|cDpNM;DQK(GEOZoXa^ zJ<&Sozf6CyT$^V;PkG_-_hQ0$R^yq`)VjyD@{+)tKhsR@C#GT}X{53wNbf>1pIQ4oExf-nqU5nl&{A`p-hlw-3a2jSlT zB@1-^iJh|7)BrK#T#@hDqO>;3tF!|HOC;=^hl@V2C#67zM3C|0M#IKyB3cS5Yf(XE zKGl13wX{yYRS;{Y>#4NWlf7Sa87R4ro|eu(*|w)Mx@DFsj{m78DweytjFSE>&Sv;F ziG&^Hrroc7i96a-ccEv)!%!+AmDx~8Tf1qjokL%AN?dA6Y6?6yHDY2QM&0lX%2SBt z%Ejgr7s9MHN(Md6ihQjmQfh)Tt*L-=8)!G`gBTS93oAMzQ}t5Ck?A<=XT*Zo6Mvyp zAlqPaQ^4_hBAbtS=C<(O>Q7CfAk}`HAJH>kpA!^XmV6q8Q863$RcY#NE&S0P(Ouvt zw5qQy*&~_$GBk8!6ZN(sxnO4z5pEA7T1OP|9|k`5_b?2ux_~I@zqxL{BKeA zoV|O6$;HmQP1zFi)QDOd#BpnSl0o}6e~Q#TmrTn7DT7wJ*zm^owbi*lory@|6FPF)u96suqc$w?>5VP8KcYM=;h zhR7J@f6<8ud;SeW>ma|3$Qd2Wt*4~fOyQwn_yW6IZ($%RO>hvmxRzkUT)0KkTuP5d zm;Bi-pI-cEAWDKtQtqb!iqAbY29qed!q5Ez z$zBt-vNqE=!- zP*@LD2zE3*9YCY0jy!N^3hni|OG(nulyuis>f-IsY$QX@vxBY1z~E%VWUnAONsezfql>eSR0pQ`#R1 zoCh4=7-*PzEgog})(9d&8QvSIzxpTxGbO$7ZYpo&z25D%%LYsvZr$yFIL{frvhsY}@Hn@Y>3jga?HK(A@)2|$^g}%0 zWW~pj=0x%J*qCGy33+goX=u~m;}Z!T#QZ1k$)!wkDZ0Mqma3MTs*j(XuZl-aueSf~ z*e8$w8~9=I?VImiQreQ5KFfeK5P2XeBjSLxQ$F(Cri9k$fegEv+w1!>IUqO~l`9;@ zq|IDn(=ieoCJasCuQbqfpiqp(a~_P9OUbeMiz0)g7@kDGn;xT2Y$~A&BmyoUHKwSh z=_p`a8$4%I!Vor4r|_UVj5$oc>E55OGV~)Lz12D|EtCSOWloO6)O2EmYPudTfxAV} zB;&|bWX%Fc$Y4H-YUItsQ1PRg*e3pb22x>?#b1N=Qwb)EGj*Y$RZPYTvp0~XZ2Lca?8o2fk7o7igiUnnOlvL{mngGp zW51`yebT{}O`tTRMs(oiCEFaLA8-m#1VC!eR8&>Ct9M_8)sEx6e$ZY@(d0C$X?>Ij z4~|MKrch9ok>GIKnPwvtteq&f*#M>_C0-~iHXAXqdq{UE7$zSX+wx$v*BL7*3>vNg?(3L2&X(R0hhZ&e5QXOP|F(o198Atmb`5l zso(H)F-;~YDLHu`*l51_{GV0jJx=DJlPi1wjewT3=?4JxJnyXi@OpQsE}-dez|*p) zxa&>3omtJYI#}S_RiSxAV{&cTocPmN#-+T^_@UF%#^e!Z~ z4)YC}sAOkArSajukGk{>wyCKJlH;W6K*5>}WhTsuWpD#ID@+C#9$w^=#EjBbd+1BN zv!fxXD;*W>{>W-TpqW+bF-RepOW#v*PRC4FA+Vcsn>h%~QnelT&~8vPsbM_qyXElc z&YB+!&T~@W4NCDNTdZ2tN?4E1w444`G1b8dpmTTLi$jK?-LxM8cT?9 z#OY#xaOm9aitJTus9#hJ2eNgy>=30P+_4 zs1XzxDv0y8pb!$bQbEj5C5H3%vtpEl1>DC~5 zubB)iciWA)MydAPN<>85<#2;n=D6Ch9K*1 z+D5rpyuLQM9@@OURg4pBKs=r*N~!a_%iDsg%_f=R<>-vBdH#k+ztO9hz(V57kJg}3 z%wlVc2T@Vg)!OWVu9JOhX|8;or$-6_mTES3&EMS zo=wRO)kA*Kh0TePFNG&r>+Efvv%fld+X>3GHfrrPQVA=n1~OoTkxo#;-~o-E?^0kc z%08HtgRYf`#5+_eFE0&sFKkfUo&#iAxFn85>gb`VA-~52(UE8?D){EZ&_0dKJT5TE zDw3Otvu{q=Gg44kge7Seji#|RezngiX~&T(+}@*}Z^VDrA2#;ClbTJe!Kk<{oDOh+p9(iLzqjR>nRv3TwAp35Ppa z@>aR`wI{JhMD<>E`L2h8X}iDQ7f2Y^Zj{7I!i|4d9YHl^jDfGMEs7T5Se>CTH&@ZX zV~O^2mi|;ZLDbg4YyCP_2Q1cIS!C}}=k89BWv1BB*~!PV!lb4xXfB(OBc)$eM7Lw& z!>K~n2LCdRN9ge8rvXrAoilMKV3eKv>SZv{4REE+%psB15Ek{V`y`srM?y|=!-El9 zDd-M39UlDQzJk7;a&Vzz#;TEu<}4Ydau|!SX18ka=itxmTw$t_6F%Cg4;3)9mB!bR zv0C>X2G$^fJ8c4Qlvyd#)K7yjNv3Ljs z<-n;;v0;ffV0`OL@0^UgUP)cu7_}CPwf@S9NOw%z=puOljDlp(Yh}-GdVN5T{?Ff! z^xw_r0E68eSn#C-&ci@9)xPM_X7Er<$qzMH@F3BTlnVbHFuS-PTh1GVIleBU!KFj>e(0$_nFze6P#8LtebaHl;WhOq0;w?bm@Xh8Muul<8K^r9GyFEqK zGQ^SEX_`%Y-p)Y;L2!MKac~eG1wH|H;DAQ#V{Elv88Zlsgnf}H&nFiHXFU-S1^v<6#xsRK<+Ng-unjei75l|KqSrHI6>;Y7cM%yr zMWI~XM&GgtOY3S-wByGP)Ulc7YEam|4+(qEQ{=O&F+UZ;Hma3ZT57^7r2bWcu{ohf zlH7noUK?laswDzSG0~fi%aMB-7+&nO;jV>b#c}6Mdon7RY4U=2Rn}h#g`o{fonE{k z0!tkh^HB#gL>2#?$XJEpuh1(drS`!TCX0Dk*iBAWICBN($NWmd`(O)QbA zSxRWfhz;%I&&8k_;qcE7H+6G#u~D~*pgWJDQE6dVY~fsvLnsbajEj~7hu2^~2+9d+ z_U;cs9U#mgUY_PHp0umqfYaGbxv94F(bU^LJx?x<>+vp&i(ZSC_506Kv$F#RADO$| zI9UT%!nYP-GDjuaW^RM*o@TPZXRTYfQlNrsd1CyZQb{9pe<9!PB_2b>=DGkUFk z`ge7V7_9+OLoaSjSFYPK@eTUQZoA znJ4GEa6jCoPEY#HM$ucz&*!)5KI^w%_^eXWqdVr}uQr3{5YgaIq2pDe7_kYKqJ;iu z+gstpFRD|=!`Vu*J6 zG`dF@oxoE^)L9>9pifyFi+SPkM&HnB0EA~gFq_v1ry7c%Tex16|=z{m9WA34iB5m<^)~`x^ zuMoN-Hb2AWn2Ts65y;P69HBVo(nrJ7AER;pYjh=CkMd!8J+zu1pZn`4O;)J~FWuRa zoBm7+O@n>z(1(>*;0hgU8Y7uQa}WaqC?SfJsP5p7|^+)IVL6 zppWuD@d`L`1-_P_mHsL`T55iTyIGWhej{ZMkv9g{vy~SAU!ViQTfN%RL@gVo#HqW50Ry?Qz_@NS}T)Z^#4 ztM6#^r*rST+AEh|&-P{)51Wik4?ktif#rS*YhpBSkKD;3evl}qeoerpPE4U2r(*~d z@dw2zSS19Z90v_P$oAcu>O$5I)z14eNyzuujiETjLJdV5G*KW+p2C+IHe!7Apxl9q zfqf8CQ6p^Rz7JkbBOytk9C$zyq%&DYsvbEpBCVuWm4}N$Ip=esQNrq^=uzy1AO}5! zB~ce2lwhld!W9;VeFoP)ot&;yWFO;g59R8`tMk+YWk#sS*SbUv5P)eXC_<7NsYY|T zsytXn*4Icz90dD|f*;l`^?}ZOC-K}WeYr5IZ z-QW=cTiKg$wwGF^0|MaFPb;Cxtvj~B8wFBu6X2A@-}djv{|kUbBfe+9ip*brHN9EV zDUum4UrxOA@%?^sr=rRKEUJ0cFUB``p{>ULB#%tg{M+I;kI{;Swk9t}F9$Jh{9{?- zzbc3DXVPmT2fIgNepeNvDooXv7W?qi&S&69$(DLzg24e-KMbE;T(^1bOlnE((Q#&< zWgy4?J=@eDF737H87ZRCmY!j(GmSQ$D)9Lcz5D0tfxg9JSNhArrT)R-x@#gvwv5RB zqmQW?8!g%7dU{AaIR$2cH?&IK!aW_RU^$ey%Yep|25U8oxLdxZ99JQCB@Kqra>chZ z3p_!pp;u9(wlyhwasAcjkoi%?);F*6x|6XHcdgrI(#6w@y3TTeRdP@wI=T_IZsgnG zYNk2MVCa*(!{ZxDEq5OTcalA^R(a?$0wb+&1P774cZL2A=FGRR3aN%EE58>33DKJ_ zZdHdOZTDJdP4;6ly~j#IKM~$moz7flV#Ijj4s7R* zi#fFz1`RYSePJ9`n&PQM&_N+*!=azoMWMJ}TEh_YNFKS&yFueKBtcIa!(hC3avsVx z{21ZYmj+`6gRw^M2UJ6`eVxrxCo)RhdAK!WgSAr!VRf=hsd4!inzLUboIDfcpELfIoL`PE=ictPYkh7nd)~m(R-UQ!d~*91gH4JOH#SVxX2? zARq%}W}@xaJ_!i?w9ewZPWCij7Cn8_gaa~`el4j#J;x*UPSZF{sXtNQ&6{905AXVqehbHX zUSX>54k%wNIB!_AaqgLR^^)|BpP6`X-@h8`$NrK6E5tRM$yS6*iH*^1>L)50L88Fm z{YE}&?vN#p$+uLCzU7t0IcS!G4=@h>z~CKxoVpFw?k5B?v=F0(?E_4EFi*N(!w6N? zqQccWrRA+5A`c5FY9YB5Vz*!*gO#+o*y0)ifnwrf4JnRz{d)wejEj7cAQrc@&n&3b zvXgu8*sACvG?>}hm8Xva!(7wX0}C7JIf)n5Dyf3TpF;AF>jR7PcNi}Db#g~{sP9GT zf8-XdY?t(N4a#?r?l)F(w@Y%Kv8o;Oi#K2>$y({Mb9|lz2K!dNdhRL|34>FKM`dYB z`4eM1h(S&{^}12Gt>2cOFNW=I<8>7e2{MEGX~p$1)Q_0_Anz*ls20sd7`*pq4GjG- zaI|_m2LjR`#9)*%4MJ{ae)GvThcTo^hOr3M(#rSL@X05BUQ?=gubcRh3wUp|tX-K} zDoPW%1$@f)5)|tsOpJ%!OxGB7R*aM^l>T%nfDjboFqVD?ER6Xm=dN`MhT^KW{$2wz zl->QDf|8mm$tiM|h(jpbPB$%O-(YTc3~wn;*$ zf)102Ww+2rqHnIRf<@!naCP8YMi?UFhoM+t%u7+~;51sa7x&shO(_uLM-7h@gju6- zj1nN7akrB9A7IunYBb7&pF}2>1(WqVPM9|7vD9t9R5E@R22FQeZj2K{KasbiuPRQp zws8RWg{ok?Y_eG9Ua%+9V%~T1B${6fEcf|UfIbAyt;9I~JKtDT82Q*hG7zeUlC$fp zlY_^CPyoe>Nk33k;9!;2nYRc*3+P()EeX45Ss#&hKAQTr{)3MWADXxF^LMABAOxX+ z2&=oxOQ@SOkYQ3_G-r~&siejVfwCB3Xf|NHAtNUH(##Zeijxmn5PTAue8bAvC7)Hc zB7S*;XdI57ScSBys%jb$g#(DcABaU+QB-!obWBGdAx{yFoBu_%{+y4SpI!|fRgKDr z(x%1Z3GfehI&dLwydPwbt7BZ3+x)uC_$wPX(ryG zNBs7y?RIm_G|+yZy&?CteDUH%H)HS3O6|rge8hLjd(#Z)%eDQs^cIi{H({^bL|WNx z%-mxl0vLB6HGA!k9F}CCEDxJrZ*JLN<+ZpP&$f+gW?E#F%RWqZbR45ZnK~Z&G_5t9xz?Zxib@@u_VzYA5Yr!ntD?^~&mI$8Z zyMzP++Pw)CXya*@?rO7C%#qaS2R6O`FDZH>F^XNG>=X0lU zEjyDAd-4Xlqcn3(P8)lncZrt9@g7JPL6LyeNEOY2r<7mtM_GosLi(%>XaYJVBe?!YJ&%s2Kw@m()$^(u zS>Nx0;4LZwR(%#&Y$YO2sdE5MG=$SlvU`Hj&pq+ZLaM-RHzpiamNYPKWwvi_FU*6+&1VPDSgCjQH)S@J!p z-!he7ReRZAjqjoG8>b%35`5aYWa%$sfT2~f6`i3=Ct~f#CJ0&R+$Q=k)4E6uN{D0! zNE^y8p)@*HY)0D1Eq^yKL_t86$E9QH6yYf3NGw26G2!_^fO~_G>-O!iD!eLUJct2} zk`BzVCP1$iV@9fk!m^EsH0k`6*)SL|QJSEQ66A)YVZUf`6jJ8t%M=HBQnB8(#zJX1 zqA)oyG`f@SOM*Aw( zSJ~8b<$#|=>lqY-yo;)Wp+?clANWMDB7L3*)s^9V_@80cUTxd#eYAtjNc=~U+!GQM z@G+`4B~DN<+&Xg1#Mz9P6JFvcicq(T%;S?}8NXw6#{brg^L{7H(6C@~R}ep1hk#1Z zjg3uXMLm$12O+5JJP^BL#A4-Ig3yOq3kbcXpa`KgOo|aW?7Pp=es#7DaEfyM+uMQl zJ^++hy@_-S%Z23H*{37+{J`#QeSJL{xD*BwZoe0xIE0fw(bez=;N7w(jpMG`&i~D# zCWqgE1HxYF_K!X%V~vfDf~RIzZL%h<8_w(Z&;NZV|JXUXqn18fzk5FBw>$q7CEf69 z#(7cc`uHop*u~Ks$tzdy;}4wD+m5pV63)pEKTJNIZ%9g#xRZZ-anexF2Vue;lr)d~ zewmL;Y}AYZRSjoZ#`1GzO&9!_153=zskh`Fzh*=m8KsZ9!>4{jt59Pyi zOE2Z=y(u7%621UuUOkp6L`0v)QsA2Moy^P-NKT-7AVe7H&=8j6!&9w zLxtc*d1W9f3>pqI!iW$mq2-haSmZETYQo!r5CyX$g4pA1j8bpoNCZ8|niTQ=!N7^R&cSPPU8rv=zuGNWgVy|3@qIgrKrJMx{jGZxk zRmd$RVV#|NbnZ~Q^Q*RrpiN%KQ`McZY_cld*k=q~QJm^U+|{E3If{pI1*XtDB?v~c z8FZmhDaOT*jqjF~MY@P)+p5A!<%s2A)}(Rp-VOf!wOqK8@bd!$0y4p1^U8*%CgnbQ zXX6wUI4l)CWtfOnwei%18+BhjbfdRo1L095;!@E8XbTzLAgIv&C$}QiucKd#!gGz>Ij9&!Q_=@2;WY; z3dS0mH=+T6%|TMbW6x=ftKcc?RDO!o_~C66jeFL<00Zt47}a=vripL(Pu*Xw3*H7q zi|ha(?va*^cV^i42ZXc!h1VxymP}-Z9?e@luy5TfyUhn%n2?GyzvT=i}Go z4}RYc@Z0xl81vfPoc9`SYNnlDaSHe}?{)sz!1vGD_{HVv1D{Sc`>dDq$n2TSefXD` zwIN$QQnPJ$F*RWo zP#i;4hJmy`R z>OHwc8?v+%*XMS>ofvo0diK7fV+ym}%eiSazUm^>xuA0Ty|iy1h)1(?EF=(o1Q~QD zTQf<~T2`Ev8I+|Wp06AZ6km0LtjKBd1qPEc7s zM9-lvoQHy_f2~Iz*;Zd8m+rvstQ zRQF_jANI`w-qim($?JR9$W3Q00xs_^|LfELpXihAIb(cx?|()tH+>{8FR!bh_kUth ziZzM#nAG84ua?yVBoHyaeCb--e)sCMn|JnN_@DDhpES93t@M^u`{9nXSd92n)!a$b znuvvH70RKf&4*&5*Xo-yjLZLeeiu6LKU-#&jSV7SKJ8;f0ATHlo#b}QDJJRdE@1F; zG?>zuDmGgU@=&1Gt&ZaoxV2f6JDZuFZdwstj0X;@shxSq3BkgA3zp(R^g23Jg1GlM zoDvlFCF-FZrMs@F3rslCHjbl|v{<3$tL0lNNWup#$*cle91l-wwM`_H`)5k2KKWRV zE`E-FQk2kWeMxCUe%A7PIuAou5^DxJJcOm35Q;ENl>m;K@)@tb62(-{fE-9)(Bufm zLZ~{XY%NDY)H8y84_%#c_%3!oo^I`|c+hMy{7}2qI91xBow%2CnS1bkg$P*GI$dunmOQwgw4y3e|xrLP#gdIqM!Y=Q{ z#4c?(r9nr-^8FDRFD`3_M-r%+^}m8;c{);WoY- zVQwB(t2MDgs(gu64|Ir3eJ+8ZLgOVisojQh!`U|Tfw{mIW_U`lbdyF7O23l!r(sHwaHNJ`bX8!1vdu$-8;I{2T4aC)mc-u>9k zuzL1?wE#zz?fzdQ9ij1-?_p@DIW|?Mq9wpzK%i(oD zj;Qz8jOllqwlHm3>;HG5efi5w%*+RxbGbjwbU}Xbf(Z2Pp6}0%fTopnv2PdC?Pu5i zYyIPnZa}F)=Y&BeO#UGajg~v58&{!}E*^1t);% zms(N;#tlL!NCKURC6sjlD~A#;$y9G0Fgu;3or`cT4@e6Cp02tevJ}T66q->jqb~zd z4f)LT{yQli58fMfX|jAxrkSazTpSN}3Pz$2rnyHN%JK>m7Dp6^MHDf`(aIA*0#VRN z9J3Yls`>6tJR=8Pc$Hd3YKj>Jeo1MI^Ep}?q!cUk)Pw#BDYNHWOl>D!Kya6awFDmg zo@-N?nP|fCMC?kM4Y#m}k-+I=_}1E%go+6!n%FH%D;O+yU=zj@1cE>wNbyKVpx|I? z5WXBJAq$K~1qX`MKu{#0#1nNkgm)5pCGqR?xw{U_wkHpPD|64HyOdG%&l&NN&7u zZo~H_U+&E1B)8{s;c)XcwvW!!fmb!h#St(OVSeLYeMjj}763{%^j|kL`R{mPu4HmB zVAJa=dsFuMEATk8dbcA6^`Yb$i_#>>Iy7Hg6>gcG1t#^SF&5wUzq3%5`?) z-Rr%1YQK2(jJLt8{ac&-y)cx=b*s1`&_ByEg+JIYt#RQ&$u zmc(Nvi&N;gU!IamXzzOQ(ed$0k7hL9>Zkf_D2(e3+n78Gq)t2(M67?V8k`Qg16B)W zJum{Dv#C+c$9K6asvDGs`r~DzSZH42^2vbsFgNNHp zETl;x^iYk8uF`4_W6@u-pV3cL3B(&qxq6TUBXbgG~5C;YoL($eyndi^@aDo(8 z#s&_Ww}0?)%ZcA3BP?BY3!b805#G3!GeTtG2yGK-o4I*LA5rPyVSeMOSFNk8Yf+)htgNepclk6C4^&%{* zK{!nVQp>XB&B9*ec{ty{wbSCko3}XJem4-l4Fi4yQ(Vx9-@@-VZ(FNw2)g`7_pA!E z_x(oq^7rR71+PHCORHQHS9na0erJ(p83+3LrzObb}Y$;Xnebbi4MQh073OUsLfAT)tKUISt*0Kw?hk*JO)mT6{+B$n%t)ALH< z4f-lWKMk4;eYtO(qQ7;ISsEWp!v_*7FDWnR<;Jh{PNw(59N%l{*z1W*t}F@gdRQ}P zli7n{QS@q~s12 znLM|zc`?vzL=J^^FlypibIYd8L+E zfJgN$=lbK78S}G=pUqhnw|S1bX@2&^xQz`BMm#F!>UZHLU!Mt~3M-~So}JxkZGBBxtTFlnI-2Tw97}C?%hRncbO%VUbZU~O2c+XK z%E$46MJKU2HBV7ajn(QJk(U=MQ*{HziFe|GqEl^5QoGodA??)0X=^crBH<+UMc-E> z(f-hkT|xMWwZ;p{Sr#pQN;I%?rf8{!I$D03k3~=?uxqXVz3z`~@)QHSGo1^R`MW zqC?9)476O)G}N93#wtUlE1+a|Lr6%SwMEey7MZ@@MCLA+qni?Jk+VJQRf$4!E7>H} z?U>m;p|fSPrnf!X`u*~)gKJ;)eATE(lGw+x*qm@5TlGPxE`N-;+P&gk>$o`1@C+c! ztWq}^4pm_nr}QpeI`EWO^#-P*QvorbzH175QE>p&aT;L0CTfW~{P#rglsr0YHhSd0 z7e3-MEIC$s7(snuzn=*l0D1PqyTY?0pU&)EJ%37=#c?Hk=MAp4;8y(d<(eG%VeIMk zftEJ|PcH6XRN1YM$b5)Yv@9$1$z+b#JJtU^#<<0yaJF;#bc1K1zUBR~%iE(*^Mwr` zzx9}({K|TDW@qcLsI&Ou^Y&?BSr5@T#x+Wek}d+{P~cg^Crrb}3xl&Ot9x79H2vc1p0VRADBxwj z&(IB*`kx#L`W%?={G-QYejsr9g>XVfk(iUD z-_2=X%^v{Z?Pj;Os860ICbWUN2Rdo(bcf6F>%BWkDoIa7Mg?)y&=3|_JQ%JH+T(lf z&l=8Qz?nIQOf1oqDmH+=NPk35M$#PSbm~&0WHCrdCT>N`^Drv^%$4PI!b3c+$V|&s zJU}$z5#8ffZ2YA=Qf$|w$>@vqe@IMSkLF)ULQT#zHOZp6Q~7y1{9W4K+B_J|8tHnq zwHS2F@$|>WyUW~_siTR|>2EW-qb#jH=8WWimKuSzP!1L=YsBkG2X*j7gi0*R+6;1C zpF55ZcC*|5wyQ_BgNMidzkF~hV6~ASS-b!|U%>DAPhBf6fxol<{eIiE@^8s8L7KRv zI1+STA9R#v?$v+&-RIU19v`dyTP~V_FHD$%<6*Y!8b>pAo2*P@=^RmOMMCa(s7|-@ z-wV{8q1@BEH(Pz;s5zQylNocA{a!Afd|q&K5po$eTBG_ca&69SrnI_kzcA=%XxA;P zFQUd~>LBO7!gjmM{Eh8^-sSvjNbZHTWDwVh)%t3kqK9E%QTo$S6FK3ch(!j9O>zQV4bcrO;E%2Oz= zri9NK(~$IWmPM7*YGR1J0DMt6(G%(Ho|?xSN|IUcUf#11ha})wLNHR%RJnP~IX5Di zEW^@E`WhGL;d-R%4oszOjv9lU29h~2mqq7@&`QPOFOT(0X2LTMvxCl^0)J^4T@1DP&9+x8GkquQHCpq0vCfhU zF-eJ&F5LT)9t-t#%b8%cO6An3Vb@V$OYIb`5oBRjMT00Ho5#r|ui^WTbUvvp2})$_ z@adtIlbeqVE@3)57Si}4kOR8Feks8Y{z#9*@Hn{F*p9i~(sp%?3&&Ing@F_?Y_Qc) zZ{yTrp^8uxG(}o9S5@nJ=*QZycK87+%STA3S=2#E`-BUu)VgV}$q;{}jfFP&W+URl=}z zXxgw;BnsFk6Q0%FJ1-XGp!qScNGTpoVGjbbiWD5wyLj$2Je%bnZ=jh^Lu$*RjWj2H zcOi_-fUkyDvJ`2JW3_}U@hU##Vod02s{HQ$V^|u#jFhF*zBy7?AfzS$yngLXcEhvCG~j#No!<|v9Tpg<7io6_$0zDYnqq#3S7kRe*8h&$+xUdzV>`%T z6ww9jz&I(s>n>pzl?N>EN%2ArBS|QDWuo^qqP1+IR&4hy#puxyb8l7|eEELYd0%_rupkDaIH&%|QxKbSw>9QD^FmgOYkO zZ8MG)jV|X13+*!)^^M3UsUviauEG+?UlAs)bGzw5oxK zvBqptZAy_-Ht>{2XqD@j;Z#frtuOd6I13B7wc;vohkmKnay@ne7y#=iY_WX7C2h2< zSgyPG&uprEaKg`ABGM5<<`ZmLE4v|b+oc+>^QU#KAK$!fQopfOo+k_y{%~B$n>-U# z8ueK_G_Yvnb>{p%UpZ_}_VqO1RaT1ED>Bam*@c}wmE+NypDX0RzH;|$o$Khz>q4P* z5DxwDulTJ?)IZd}NV=_x7+ePpJd-zmu#a<~$0>v)`qIW`;A1iNaUyCqxYVNf%cxOq zELfb6!l`P=?X@ZxV~ne607-E({<=4svFYI(W8xyJ)NuM~AuRGhVSBnXzt{e0wL1^5 zIb*~p^OSb>`(JA7-#0hEsS{2J!x*}>GucBH@W?3xH7aZ!P){F$73-|4&!hn*e|oKA$ya!C$k^cY?UhF3Qh=KJI5c zD$bHwV2he9<%U{G!+TP*o|()C0NbLoCgJ(&*4E(5tRR(!DgTX^hqPKQqdEJh#bW+j zAzjW#xh@QC$5Yfw`#;Gm0B7lHhnA;1+j@ZR}$v zRvELUDX7NDIIvJG?t_6|(GZ#$l&=Wa2{#1Va-$^$S)8GR?%WQEn@R?01ZGZgy=w!Z z;xR-Bog|rYTJ89R9!ZnJUYiq2hH=Vicv80t)E=lYNiZcq^qK7oJ&fyw?TIMwmp?v( zaR$1=gj*`2qhR6dJr=5v;4COmB9K<7UfLj)iZvzU`K=!_%Jy-X-KY8Y7Nk z&=dxW2DmDZj-`rdei4x56t^~xFhKfYb7e~xXP8s4TnF4Q5CcVg>_fa?>yiJH1sS7P ze-7`|xJ>Ea`{LeVW}>lbWcDZ}!0gfR`Q|J&+s*Co2fml?G+v7+W2OEn6bxa)uik-y z^V~$cby-w!N4?pPE^ScK>&Rva$LAk3d(vxR00Z5nhpt_$J*udqf85C->8XDa!R-v^ z|DA@@pAF}@4UbheD02k3?@I=u8a>YzPhLDvwRcUMT`)hmbv^{XyYS^=!`pH_l)7Uf zi2b3t<36NfE+{>-bU{qC#di07B-h13yV#?8ubz#ZCV|6|pf`tK&z6TnXWgbiJkutN zhvtP+4?bsXaynT(zHmB{p^zgoEm_!f5qCxZpuH_E1Bb0e&RC7CU03{6_`>YLOhcno zlzC(zu)XHP3|mNPBosOoccrS3-64+(9M80gg)1Q_p&F>y{jmhva{lc2Slatap;d|! z5kv|Dib{`-oy-jVygK4aCV$@HBb6G_$quIG)@Z~;p-6#Mje0@fy}GNdKrG>&D#-u}tG&7sQgS4sSnZpp-y9bj zjNtkFxR+4}sRRi{OX{radyRO_XO^g`{0Kb!b02g?Nz%^ZD((-4iH zSXxP@=fNOCzqD#Ja(Tis$Yz3_E{~-!;(P1coaaOXoGj-HU4&}GmEARKX-tPe!!217+C|4u1&`!VP$%Cb@ z{+o_eUnTwiZMP+HE%0b2;BancF>fKR!0t`18R&!P+cUehpkGp6zwXAKuXg{MtWh+r z4kQbld+61_e!i9K5){A+I#`r^6xf@_FWA9l?SoD%fiMkEv^DRq29Kjm0zJi@B zZTL^@&&Wpe$~RxK1dH=k(imwZh(zs<7yks&Y7&HpdMA(hAXSB?(2yVCo0YDYgFg^V zhCXFu9tb*rXNFU4YJI|~B(S5=-6=e3yWz}L<^0i@&pTGnG^9@|;%-rcFzn#h!z`LG z1zrdc3;FrAy;2|JqD~OG%g*@ttv+t~nVE(KnWFfDAAG5|BfERGvUb5w7omj1j`lvT zYG@)a>}FSvVEH9477ZVf&_nvRLWON#UT*T3WY3aaHae!hD&OebGkkRPT=h!w^7$tg zp*P z)=B!aOM9r$tlbmb)aaF81N2nyCdEa$LB179zU$`{9;W?m;5EZ^BiS_I+T11DOMQF+ zRwL{=DR+)4FHT3_pXS^NOYu#=$I=f|CSCUZGyX7r*1dx*YwN9=plL%p`JmU#VaF(1W;_)N84}%pV6+mdu>FnHIX6q8L zHMWr91SBCt8wN!KWym0Xa1KyG-pS5<2Zf1n6k&#QIGOb{&Dmy15>;|d?EHXC;OK_C zoag`(UZT51&@a~V=<9h(yTu0K#t)T3F2kDjws!JE!^LCI2AB7X4JYGGN2edo$%zq_ z(!*BD?Jd>6gFG z{U4x=WtR>pFN-LYMRmsj1wGQ$J~}vD+C$Ijs<#U>pR@mWe-xF+xO$moy`}7T;e)}? zh3{6YUT?_e*uC1RKA4mJk{1(3IQ#hRpc=_Dqknbb0ohl!`afhR*)i`fpYyb?-x%#+ z_F2F6t;1$-e|7OO*Q?H+KT$jHT3-20MBiZrZ>AY&X0JK7i_!MU{( zUYnX?q6ntfnN#q%tUshsw2+jnpS5Nl4~Ys6=eGkmL1<|Ms0ph&4mx&;bdO%5+*o)t zs@qP{p_@kVTld%le%Km{ec-{HUE#+iAKnN@rd(Q)m}(!CP2%r_7Vs%Wk$pfCJQD9` zZWYL>%lbU0xPB*kv0|Vl0+AH!m4LB^l;H45Th;>U9lXxTwvB>Mt^hy`Jse2Y8YAY) zjc4T#D{(xCa4b&A#7M;iGZoz2UaR*bNxxD87x~IxGVqY$zN+Bv?Hc%8@^UMWkqzo< zT+i~k>D^*XeKe|@qryZLv)V`xtcXT8qSQ@N&}B#}8tm;HtKTKX&x~PT{s=7?2ZYKm@yt&r&g8$$wfkk9FyqXy@FVE@sUioQ%f25q&cBE^BK{^o!Ku^skHDzglkx z-X8XRyf6&lw`X_S$IQBsyEDqSo|<1f>+!gxFnNpG{AlakuGM<)RrR8v^YqRhjXQ?I z0A@V!Y>p?h?@1M2x=iCZpv_%*qEwVSgELsTLL8jA7N4i;ELby~C zexJD$#2bGeUdAn*LR2P>aym&qYr>DR%M#)_E7P0$-j-+8;U8P9Bc`t5%msMpIGk<- zn>xWjIB{`auw$$NZi2-bJtj4gejn;85l4VTm}P#1qZ14&tM~{lUsF<8v$L)_MHhGB z?EI{nUyW^g@}GR`kA~>cPL8f9lhW{O!qR>D{v64VI9S1>w)#jrzArrl7@F zeY0eq2(4cJcB~(-5In;`aF}GkqB*3X^xe34>k`tt)y)iH4CSW?;R%7&df6WvgR~{D zttmgwc#nJ*QJ(NKPsQ6o1THOE_p?`9gEn+qkV%Q8e3k}CDQpAST{q+&po zc_b8Ju4&=THuX`kI20_*qGDNT6NMxLStE(IqHZQ@JrmESN8X5H;Q5bzszyRC#Z{c0 z^ed~$mEKu4fY+~|x1ye5S-1;{>~Op`t2`_m8jC_Q@h7P`=|Dm%HK^4li@ItiImsk8 zBW6sU?R|C^fn4$V`R;`R$P9Q$|0|EaX@xohDqbf4fm}()bwKn4z;rnNS4qA`JU;uD zuRH*JV}ylniv<1BdN&dvbhYu$=~SVFs{6h3*Dts3+@=2$@m{{hHBH?h?a+^+b&{5^!oVIX z5*c}rJkf|$TRynskxiE4i=<|?V%a?_L%0U$p*iSPUyb2uRIu!D8%hK(i!Mfm5?VP> z34$u>HM=U&NCOWGUJy1LlpK*846hN6sZr8gqy(*J*Fu6(f4`H1*d++P++cvuEKEn% zo&1pOx*^r)UL37I4CYi-_LLrDO-$zJb%ji_2Xj*C=*3&w#zXRUc2zMf_O{P|gt0uQ zX*AX%HP_V@ppZ{y$LE`mR&=sH_Ffj<++TYRKtDmFUp_pBu|{FArF*i%iP>cXhbRB% z1-P}LrL!af9~Ur>Gb)RkSJZPBe1oB>72Gb+e?kT+5JD4bhObjfdfUqt7ZbHV1Pk=! zAErP+8Vs=zL60AwVtWPZ?=er(gUa-k^*^Q|wLuw^^6D56)|gZ`NFP@rG2)gci}}jF ziFu4Smg=@AT0tC6g*$mkCJEH$?czepvF#ueykE{v-uLIKc(ym7437xyadfx4+!Z!9 zoYM^seOI&JE$&M(;X+jh-3BLL5B48 z%j+(Z588YN`rG_exdN(KEE+i@D`21q5$vDdrvhf)8^gc7?)<(^SGtjTt{&LEzg%7Q zi`lB~apd%Ye(TG|!Iqto^YsAK>xm7t3w^raIxtYR_N+wRUVux^~Ri4s52A!}$q*^+a~?8FZ= zo`{9QCAy94;ItINp+sv24t`o2X?TUn#K!ax9o_H3$6IxCD;#mrT~!Z!%(_N}l9iLp zZonbzo7a@%seX>%(AyNO%2b1JZR*b7VU#rKYq+Jhnfa$#mJWm#TzI+GZbC|$Ar+8? zJ;!rX@2apc@Y>^lhDdRymXtjWbwk%GkVgSjFEl7`+E%nIsU)iufj2O)T4Cp}$2Ls5 zveL|~SIXPD(pOjN&glx%8rY`1DbXsBejDZY)q>(Q+3{{nti2Fv;R|`pQ!?L%pzDS>rw5Y`4 zY%^p!K3Aml@&Z*BL$AmuA8~#?zJJ#} zwrcjNbo7rPGcWHw!dB14$d&tYmV1feMhp$e;^Y3mC8LE>H3#DEq0px7VSGuR#yg}Btxv{K*-vsf=$E}_u!vAg#kc`6-eJ=RINV3E z*SSXaiEJC+~<0y$MR zbhC$%VKWk4iWON^RW$IitKQuGj$@wM3b3fP2T|y< ze&8kRf4uv5#kMZvt!5;|qfUxj#V*sQb{r-eO+hkc=EZ@l9^Mt-h;q3HC6Qq$M&xM~ zqJ3dh=nnTb-7)tRvS22m$y|X-yVc+yfBBS&4fWByqtehE!F_F!FEoH+$kMc?ke{j8_Oa#uV!>*m>-W(yPT!*_k3wCws^sR33uys;(X|Kbm@DG-#c3@nMr9L<~3s6 z+{!*K{_)=;GZ|XYZUCfjSI;PTvCaL6+r#CwN7?IiQ`3H1_Qt29Z)I;>$4#ySFHz03 zOFRp8HX;c(sBIR83e{a)QND^ah?}o)ROMGI9Rw?X__Jxc)LXMTtUYyfiyU5Q3MSS) zHL$^LX1m^mae@!pr|>d5zR!ulQmw|4w_lNn!tiL?%;<_vW04LNl|-_v6U4-EN{5K* zPN&yU7ZP4j<}7nSN~+sK`b-fBr?`d)`s0M^L$k4BLXO@KS8Ci{QMEIe3(apjI{Y_e z{r4heK`$Q{ha}KR{{tyqN?^lDc`u`rHdK8qqq_SdR4;OsH{OcmS^ZF3=BNMy1zWDL zOpwz$LSY&v*93MXrKrJZ8V)IvAzy6k3tFSfkI@#j-FJ$z^H&o#Xc7>IgzKi7Pw zxw-Dy*AHM_aqCz&$^vobrKe`m5W1;abTHAn*8o_o#DPBNahzUkwffZv*m&G(ne@#q zeRsG&x86`JTpP z1$Tjq3I5{n=I13Nqg8zJQT#AN(r$PFz7YDmmu?|w4`SgVJMucnc(q?MKM2yp`rnQH_EAI64w7fLe_h>cEWvf7rb`H?yk!)ehgj{kz?Rh_p*S`})#z z-h2N|X~qy1caR^be5HuM7>l@8so6^)(Eont$yHaivtscHUo>|qE6x!6hig0HddrBS z35!88O)wFiEG0PCd594}e`&*GsVNS{ez-e{}-UpNI14{SNL^$;6X zZ{{H#jeZVinJ|XoYH9Fb^;AJ@Ab|+Th7r(Ep$B--Xxoa!J zG=B=|mJ3dJOM@nWM~#(&b^o^O0Zmc44RIx?h*(QeA&v1bm9zbVa&h0FY}7U!@v)@N zU@pXm(O!5COSveeKr~E-H9$g`RT+q;^PqK=#C!9=;lWS>hyw(j7Q#tzs!)J2yl`EL zYEyL~P4hYm)^hu%-X(*r7rerd&%NP-!Th(b!1#nhnYzWT;;w5}(&iyUYpi0HLW=W1 z1&+P(e72~`H@C0A^(c8lQQyhy)vXv{N1rRaU)xV>e5YfMmf{-9-K-_qy?;No=`N>; zgCeQDB{lFPXQo*MB(5Oemq69U}Jh&6FHh6A2=r~HC;!;dZ z%sNmB1bn$+%eKRJFHYHW8=|2~-eXw{?m%Tb_`|f_%d?CT|B~75NxwJ1k(72E_fB1I zr{+0Xyg2Fh*jxiryG+bYD`L2jE?Pbu(%5I$qD%%RlVx#GX= zH{2tSmo_)_&;rR^&Zi>$E0m=Qm2CSZu|P9WFEKDpxb|IA~$blhcC zHfUlfD5b39q`N6_Aiyglh5?0u5{OC=*6jlP@~BMNTP6?;##ByHs-&VmSjg;e;vWR@ zA{YEH#VN?tZbqckWZfabwi9CcXWZC{^@~~;HB7{5)gqdV>0@38QZwvP5O97cR}Aou zdgw8vGW?mLhG`Nrw8(tt*)RhY`E@rf)#>Uo%NMnMwP~6(aU}05xj~Z*92BAmHNa4U zqR2vp-Bu)Y4aO}7l_b#iI7I^}pM*G({s%772Ky8OuN1D)Uh9;Og2_aeiN7(4a~w_E zwnVE_(HQguz8d|g_A7h!j}wu^0je49Lg(4m_YFEGWGD(y2z1oZ+s})OX_<%CX-QPb zSi6)>XmfR^7l$w8Aq^(vV!FlsHn}-IcBq(meja3BZe54=rrUG#Bh0C;tf?NGxH&E- zL~?s6}a6)NA+?J@QK$Go2B~d zR8wtG%8?xDDSBM#DpCZ~hfV!Y$8>JB0!3Ed01x{<5H7E%Zu3n6UY&b!E^}{P7XSSD z0|YZo|9hKOP}*$lHZJ#E99LhQhhFAp_*(M#FnMj@R%F#lq*lx?b4%+-Ijyhha&{M6 zY#bKuMy{8Dv`$VxD+Dd$W;2~*sHS+P{TD1pPG0t1^y;02cU??fuB$fIMg%N3o1Yvy z9}at5J6I@x65zcx9I{LQ;6-J|-J?>;TX)_EtqiAKdi2KWb>SE+p=p)^?z=Hfep_7HF|zZL=L?BO1Bt8Bomd?sX&gAA}}Rt;-<+4YDf zBi1XpLS~p|ai=l{8nw7mLGTx67ri0f?2LUa3QOUo2jCm8aE^LQk;)6Vc-TT@Ad&q_ zB?L^u_h{8?SeikmGwXs!9K1gi;z2kS7z=}zuC6VXQb`xAgg{`ZI_Wi7Kd>_ipjjF6 zn#8mi$v9sNzv^Ip_zRS$tHK%QH-=3M3&%+3=09x_>3N}2Bs^^|@UlKp{v$%C37J!` zE-0Q>tfrz&U`vQ68t`_OO3+votW*_t-h41bL;K_Nc*(0#hXy%kJoAl!L%Wq)2p4gn;@;{fYaj~(nvH$7b6)mkZq}Gj0{;m8{sYkWbF7GN@cJDe} zAq!sIK2;rE_B($cFaoLn`3FCh`t5GeMr~A+yQ6d6dGFeIf?9Pzt?XK)*d52A8Lpj{ z_4%y8^&Kvrv$!?7oxtyVmv9e*M6I2Ux5JHbl zwI-1A4N~h!d#@zCXez|7piM@rrAp|nF5XaE;do;o8YmK!L2LF^5*Q#PjOF}I;b?JG zF8*09g8qiIM-iZ8)T$UYYx*wvdG#NQ_;Tg@+m(eG^WlWwe(f=Y_*1zCTX1NqE=y z8}1R+zy?+NNhs1zwyKynGu5LP)WB*>*3KiGgI6;=oOla2GBylLANabeaF91h3mMl; z!mWzFE!|^cN6UI{9KPi)v_M& zFg{>gx!VJKEahaLde;B^CF9i=w>{pBT;VNm>&E2-%Ng+pj2nF4ew)wh?8te#N^H#i z{mk`Zp>_OLQj){&g7ln3{`=vC#b%f;8xH3C34r84D@!K z0(4s)s05-1(O+KKJ0nMb5sAs#eaIAYts?S(fmf0_5ilt; zuVWCtPM~qWY9=WPP(>bGiSwigjR2XZ$%qc5TW?kJjdnsgth%5e(3liX#r=7lqLb;^ z7ml`57i$Dd*KRG<%VGEKR_xo3~}g+N?! zsBTj@JSnlZd!`Jn1agX{=J9vvjpzbm-z289JxP%+HKt%|o!lI50xzUJs+oJ^aq|4iyJ-3Q^;E&#)uQQ? zh|FJ5s1gKw{t@2I4s&Rj4J*zi{@b(UnuHQQY(KfBRp7MD^#Q!Jd-q;7S2g}!iUIs+q0kNh7VHjWm3h1XWNL2Je}*S%!|MQPGlc8O z-}g?Fg8BVsmV3uA}IK4(@@@UL^n!@ABX`0%{cB5JYdY|<*d2gT7 zkTxy*2SSS9_o$BvjX(b~f#^@@K|=QjHLRbH^1M&(R8bA+;S&NOZ%}|BZ3+Y@rFD+h zW=kAKvuclt1s#Wq{$TcpI#s=D86@f&hh%8`*=jksi8n!CCMdd>kBDx%!GDFO;;{BiX>tqZ8g0<4|<%Q$LHO!!U zBwDf#Gq%>Qa4MZdH%z)TX9pSNrVdI31UBZZnzSgmZ7}3V{*8%Xf@`4$frY{?5dkpJ z3p51<#vp}62PP*PHec2a7xt%Ckvd24vODz_TNy#Hx$zY@Nt2-@?kiBNum{dVjr(0N z90CXHnZ5b^71N}feNr}RCc-sWH!FIT956e$N8wWj1>h>$%~&YoRcs%7mQ|)HRy(>R zy5L5RgXgttnK@6cY2l(PGirr`1zvNK0TX>9${ILY-==9zkC|x3(^mKZQUtN^`##le zuN{DQIlHu!Sc@8A$8cB;WzfQ||9&b%L4l{|HO9xl-%k4M{+7R-Amy4;1+AIi)*h}L z&3o{cktM!Y7&4yUJMu)nC%W%0U{VSI@~Gwi?;4By_SAdU`7Xd`o$rp@6ukD1Z3yN$ zc3bZKSy1HNtpPEu?^5D-syt}<0xx*Hu4d!8$Bf&O_gF7Y*%onLe7_eB8iH3xt z1bHK1BLujqFN1%n9F1R9WC;>~6V+5}zT`8C7$^ZU$*@2weR*^Zpn;yAE*u@* zi8(ps7b=5VB^Jd2--}GBvf{)W)3|Aq*(WGRUUrGJ+R1@0AdtFdL@-(rjSz%|2*YKm zQE4hX9P${=)VH^meSo9n_o|&#_5}~nV%A8acW$0EgzvVtvndKmT z+axtX5d$?O>rbqlc+3HZ2(G2ox_#SIRP;dUYl6w8 z43l*J+o^&ybTOL9+R*4Oe#~Ix$2uf~UjH>b%=M6IZ+gJV+B4{*ia zU~0*rB!m`P6Y% zTp|7ZyL=mfbac*iL*^wh*jyn1O<+9>1QML|_>uO6Nt8-DP0+&g8i#oM**_kEnFArzrPgj%krd?nh6dU9Olr1bab>|4ht09 zJ(MLhD#WQ4j{UGUQJv(3FINgiIZ2h++D$%|M>yP?U{`#m!Dc|I%K9z3)*1OE2x(+Q zrb09$subDu(?f{C!prQz4H1grP@;uFzY5E{mmqxOGw!SH#wL=SuF{F=k!I3n4yyso ztXJ$-xohx0%N4Sh&VMcR+}uNG6mW^LIlCGlz(k2)k2f0MXSSnu+}}#eqVC*A-IA8vYh zMcp#rj2b$}0_e28?A7KpW+q~i8qm{dEm*X+{ zZ?kXI_ABMTpPpPUNC=Pj*90;DZ$GkJ&Mn{HB?cw5RRzrO*=~y+(N{aRxQ9013T${) z{R+Fh5f+eINv$Prt=AAMRK`ulPQj{Y$4b;SmZ~u|%r^^&(Zj{%zO0=7zIs&zb+7W< z^DPDywddVPYb6YQN3l2_xhiOvqs0X=CRmsyO3yI)1{Rf=^fAfP1c?8F#+|(9GrSG% zrhjD2h<}wXfQ<_VZ0J0DkVY`!F0_k11#2;JTf!1nK6?(q>OO<`s3$W2Lx# zYrY}k;K^c})5!hHYzHF7cA3ojiGm!UM&v-ir8q?cTPkH*bwze)i}xJo0xduA3F==oNY1{J*T4;wFkR+YRLnmuHyBQ|5- zP(smpx$Qd_zS3>qXL~y0A0~bfpbb1>NxHA216YD=JWxLW(Z$O4vOCEy$@_j}xTT}x z@#4+_#A#ck?(%%%VuQ!um>UK+9=DP95{p|`?<{6j_ZSy;k44yptL*R=th0wW8NeK{OBV5P8Y>Fjf-!0#Cfb6^ zuQ&m@bhE@F4$zv)~+ zAG{t8QB9NdhlN05^LrouY_xDn-s8pxR|)9WPIf*i0oL2?ocK7{Xkr+}i-IVRVOji@ zNa&XbVe_h!jt!sUU5BL-E64hqxBD_Sc2=xs9BcVh^)Rs^pFpvo;G*bYiXu2PEJ{&> z#pH?!r3!^y&9ynx+x_p#1zn&z+Otui9Lmr*2ayP<(`cczh+GxgsK|8l*`!C;WLHjJ zil|_;SL&JJ-Q~}YH?uUBK<(^Ku>uGfYl0>}GkXH3gO-7T_y~144=H(w9+76{I`Fw8 z^{92$Yx_&v{+{#gx4XgvpN+LrKe3AoYA`WT0*^|^8x^E=-G~;U# z3N$9uBvr<)mtA>~d)ph6ZhIX@2COytlKGNjNp$6+!~f?6AedBGRxy5KDo8`b5sMQ-gu`g|8s$RG{+y1~e|C zi=8f5I21B0YbY4@0ryo4*PtLlNmKtXL{Z8AF^yOTfo~95IaN>fQKEhk2pYeGR3rj2=onAQLW7G_s@#Jk z4A3~bqP$p8ZvpBr2ejvPY9cCBQ#g238aMPD_f-C_S*pC{p!8%yj*hhNGE`jim8$C< zk)>DQp84(3gg1%rIrc0}ZwOjtJ4t8d;Y8h1ZO$IK3?^XzXrqoLo?=t^kb5l}jd$iX zbpind`P)C&9wka|-c&gFp4)m7{Y_jfLqEy;z#)|n91gNhlyWNGcskNT!6de2LUsa$ zi0J73Y!|>xCQ74vuIX@nv;=Ag6U%-+Ed$LsHq;@RVFrq#CIBszNE=_00sj-JtjjAA zZ83(>;OyW=Tt%85s^0Z`x)^Yn1C-tX!$^-?Vfv~5%j5BXP`G1I#s5quxgT{W)OhCK zcx=9s7pp@y?CEW$eW<8UlWy7%3z{)g15K6fxi4iHeK%H4{RG6`gC#-|6BVPc-zf0O z<#KBOB<{Wb_}#?Tn;DbL8H8e3ob5;IN$d+8!pJJ`c^-!OlMGP>)-oF22r*&(<1K#amff*_KGlPCu4TLSQ@dK-S`pk3j|@vF*mGi~!)H zSB}X-msbg|MhE=844R-=L$po4g@7mYkmgkR+2~<3G$9}>LA`A8SA(4##rph}--~#* zlg z;ucFt7#NBlzn57Iyv#UF5)nWYM#ioRHL90}cw%c?GVxe&vOpZHZowviCSR%i%~ux6 zCo!Dg5a&5-JWv1#>R@cqIZJ@yDasufTwDHjiM1_1za6*>R5Su8;YV?ghk?>9|9_Q^ z3oS=QN%xw80-VL)xrgKby)m+>C0*v5e}9YQ;fahsS3-Qh=iFv>xb}_zt|{q)sS3ON z74AM)2GZ2ks<=vsMI~%Oah(lZw`4Xe-F$K{Q7aqRH%U23>0}}tQV~&h<{GkA{w`9F zRlZU4;eNp=;fDL%v2G5DvYWK#Yc@2&^L;yfSvO6c4z{LsMi;{FC-&=^gmfc@)4y{( zT(O9~PKCh$6f6RiNK_#^sr_Snv?BA2i31;7E1U->u+s=&%Lu}(mJ~8;v8sZ|-Buzb zMwmW81JYmjy76^m(A{;)&S-aKja1IfSPzZN#CC*6FFX91L+YE%swZ}yo}NQ94JP&B z-DPBanWRrz?H0&VGS)qAmPSsrig;HQ^qY(~n<9l%B^XhO(WV7qP)eL?wV#b?Dhh0` z$<=i~X6#79ToD^^RlQJFNi|r?B_EeFR^jZ*DJOl+)}K&(KQhP%42OUuh)0qkZ)`o7 zd4IR>@Atx~>MW;M=F8^ZbC`Fv&d;8`^qAYYX=`g&Qc_A;BXkGOskD};97PoLkdy?p z($4YBrl^bzL?1o$smfiIU-yiG^+ZWIFkaEXkLy^TFIQIj=iIaN^_A1oKgb`{GMAwC z>=v}2F&S$%9<>3s{Tg+u{92OA)>Y0@oPZh08BnB_|Mk%P;~xEk=JwTXZ31LmpxyC~ ztZY;BO-onTy<)L|jgy5#1>1kU5Z(hf-g5u{#STWMIlo#@LQVMxs2ENXrJZb zVqHzpeUtmus&S>v2PU;tq-b`iJSZB(Nx@_Z%Nxv;s?EnpgeWU;i{{hl8_SNtn8hvLS!f5_rA~L z@%`^t|CRE1)azdF*X#Lujzx2kuNIs1^dOkDEwoG|4^Pg~AYXgTk;uFY3m4Zja&?2+ zB(fGve)*~{^DrwbXSUV#_Wbs3?Tcjzz1dyD?;#M1{*dD@l#tuY#{b4%=D@Dismqt_ z#VsY;?U}rvmKE^Q{Jgy6l$Gzqf&Wpi6^n&jmo2HF77B6sO#&Tez;f#wFzKEg-Mup? zP8mN)8H^lg7Rb9Okn#Wmuqk!~g^l>|=QztReV~nDQd~aSiSIZizaf`GT!VISm{L4Q zc+P?tkHPb(AxZBIz}3|7_{zs-rs{paR{E~n*Zc!K{FTPKm;&G3j*WYFr2J;J1_~*q zZ0Wy0mJ{uhAV&x=1f=todn5#NN*&TsMqO$1!pxC}7nctVW!^p#yUFsbsg}_JYOhUy z@<}y=t`|Coxs(EUy)s@~Eg6AfyP#wiT zjvbK#JofSCv+ssMLt?#kLb^32sWPejeG*S;(U%z5lkMY8Dv~kms#9(<0s`;mwva*5 z0T~l(4UVv&^P`~i+d;qvd~c7qpc3u+^j^tSz0-SpIPE$pw-k4CkWO*426(Df5G_gD zL`+&1wtnN^=AM4w|9RkKX+(AztIq z?M97mQ#gky|AmwJdz8n8-g6SYVn0=ejO-M0O0Nuo2jiL z)B%r?i=AuGj&4%XE7CUcExDT20vwDkY~M)>xw$}+bKLyp;3sTa35#58l*MQ_~52|5FQ0)f6bKJZfQ01I5@)d!{*)u2E!@#;U1Qt;H&6OFd; zGrwNPNVl(~SpZ))0)UTyYj2-|K&|GO!;huPr_KTEe?sP3ymJAkkZ5W8d)8;7(yV^9 zvVOqtYfJhuG4>6pzH)ufEujedkfmHP;a>A-2xL`97SsmXs6<%R@sxLr7_EXfwUBzj3rR^Z1XAeYQ3( zUo71wf8v$9%&tuif3&XTAj1{KJUg=}YGV#YV`d1&b$Mt({T=4#Kh@cX-g%YQ|E&2W zQ1Z@qc%jLgP9&Cm44+SBtoijtYD$NMLoG@d?JI7#gJLeUi=(GkFTw-A5Xgl~=#IbX ze%7=hQwo>mIV&w~Ysp+kXD+cNTYcY%8?hH3FXAolEVGorcDy{DH>_wH&CH6ap8Hbw z!7bV?b48RqkKja8W3Syy;OSFHPRf*vOhBjDu>p^P)GuNW`Nx{KP!y(($jh3_eEQl)vEw)Q0aDuNvq zl98iV_g|*@X_(pK2&g-J_Pg^Rr0a_2Aa!ae=QJ;K?M!pmJ_$MpdCvXV=g_>aNm1R6 zz3Gt`PY}mcf?1CNoS*ynBUQyiMq-{82nKnC{k;70f#1(p3;$NpwbSz_u{YlR0I5j! ztHjWTERTVnI=8V)=XStrexm)%^Hc+Y13BqUgSSpLK^*1TboJ>pF}us=QzW{nm?+I4 z&g8Ypf_zTf$2{bGsIt>j!>DPs znRSg-#mmqN7x&z}-z7`e&cR&w0&50R9n6;blLT%c1}^Q|E?2T27aB8PLtsgvWu}+P zoUU_IU!*hu8u2MlS6QwnWv&7i-fmonEa*@?7G5UGD1*N07iTk^+Ll|}5G4{q&7FsZ z6sStst3hud5VS&nxGRNf>eel<$-ht$Z8cB&LIxqV){L$4F%UZD?s-ERV^lB{sy)B!P;DNh66*dwCjGE854Wn>pw8sR(}Ss-Lw+O~l>4 zEXJp;fxI=X=dY_2Wwuof=-Fu~j>TWLkzeJMPcEtVsLu-q%UG9;!d!W|8JtYZ*(F{Z zp3fLgrE-B`=__9<>b;Kv^B9|B5%uYrb^6*3ZSb_375+R?e^v6Urw3E0^n-4alHWoi zRx5`t+6}=*L!o;bSiP^2v-Zixz1h!C`BU$1;nmY0z_*7A9dLfK5VRPHzo95y=6$rN)_uGc`7!R$f}2{heqJ9McN~mT=X8!w1}; zRM%#S^#s@|f3+j-hjn}d(Uu>8Yn-p^FHoO89T!;tJum%Q4}(8l&2ZG5Une@hD4&if zU!n-hy^lf@gGYdhQAt-%H%~WmZwOJMA?G)1nX%g--9)RXYV@WCL zS?I}(Yn6FPoxq7v4NB8@s)sf^3PDkJnTtZeNt~Lev(bf$@DUN6ArFlz{``6VEAA=8 zfRusW7>TA0fvI-zYunz(@oDoGjy8+xaX9-IDAr>h`t@6SyBLxta&Wry0=B-mWWAH| zigADyo|>B!E#||tDwH3W*=+nqhbm@3j=YHMZRSdE&2aw{E3Qd=z37Xtz-)Y}W}U3w z4+f*^)=hITV3L3F>L)ttQ~ps-oO7eOGcR-M=K%GC0;eNUEri?4u&%avXR3g!5L5#i!r+5D&kB`;r zvA%J77+&J!@B70yi)Sa>%E!ye%4a(-u7cr=NOAhxb2sqmSgkye0fBHg-diVAg96&S zMBk|y&=&F59>xc{R)4?7KG-jqdD!xcSY=8_v-KQPS;VrE->msWbS@)))24;M6XHlH zcz^S7=26g2tku6`U@`=X>=ZZ=ofGdP;QJiS`)~g}*l0g&sBSQ?BIbf@pKWb}Z92%S z_W1aC;W&H2cTI7jhhX&&XiI>9QI&#>;NYGSNErj~zepc3M&yW?xLMKux8k6jShRyS zZc-aoi8mjesJHejZ;g$}$?eRF-<}8`2)aT8iK+F>Qk_cbVPI((hw+5`U9`5@o)3If zzOXF|&`Bq~yWaP0nVtm(1}^-2K+A;|KJ+y;lZi~!uW%8f=$|5^hegL6sY335oqX#& z^7qn)`}(2S_06gJSlzon?Cp2i{R2#XOOMeNP*YRyhQGdsgxPfJRH!k{#0O`fLp@lc zHMc4qj0c6$2Bx#2csuGRo%#?V7!FSzi#m2RrWYl+9KW#%WUOO=8dsiwkgz!E(mW?u3N z4;Ms!yjOWASPMCBjx;tQCpXLg&;deXo^a4sO5L*=3u*ZN*S&K$jv*!2Cxr5v&KFya zcP6yb8=UmfkIh)xM4yZ@t+tA_nts64uiywJGOnJ7TeHeP=)%_3?jCJ^?4OxX+D&iX z&bjWpHQ|0w7NL|{q%Cy%u`OcJTBx#8q24CoI|FuVB)EVZ0bGsLmI{UxSsZ(`=dA zdZU;Z{IN~+y%=R!o5w(Lv|c@^;e8l&?#b+$2}^P=$n>aqcnwG?2RT8)+>2X(e-Mil zZH#L3SQ{W-UI2gkC1Q20hHMbC@7GGayZ~#7htS1 zD{A&{)%{PmCPvD@d8b?Q{JNDsfUvKpheZ_48Ny^~)6(frg$&BtPkH1X!3-yd_cRpY*>{Q9| zYwx7;ou*-C%a1?G?cV6w-$zCS!(WEfT3>|J8ZNNeK~(u*&NwNxpGpuOk)&`4?_HQN z{0jEE!QCIaSI-l$$Ue@fGLFg|5~_IVOzz+V9Pe8r^B;B~siZto=Xngt8qHF3iBzQg_Tm7U2kJofJ!(odmA%Y1P^pMLqK0PCNjG+K?^u7_a z!~X>oplo$7V4e`RY|_uemY`ETz0 zQfy*&sC_eP6oM(#=tA`mnDa4lj~VPr7WJ>r{`DkzCT^*Nlj`Ttp5`V?3M=ef`NTTLHVamK8Er+3fN^5(&)5lE)-g8U&ScdiAUt!d*b z!8)|{Zi5)9`1ojRuG(*`;x&kB_(Z4t9L%b@=f}A~$k{|o5vEQnzYJuYej*~j0nkTG z7&|{*I0p{;4OYj!rgs#7y#Z4zFs79wCdjs*2fF_12HhUC-_t_3KV;&ZwCDSG&-X#w zuZ~9mXYJd!{w>l~D}>>&SpkPrruRKl@ohi#9{nZC`iyr}0FAqB$5QR$L zz&pn5W;xv8eI3Nd8!mrX8wGQ7bE9-%ser>dLlIp_&qN`T#C(aAa$X7pa=6qu`CuiA zIxi-s3nph_UZHv)heF-q>D!;({P>ZpIusgS35kU2=O+F!+R@<^h%l>^7J)OgG4oB(S*l~ zpA?LiP^@BF<}z|f4Px{j6vjEeEZLw_GR~f>_Yf=-(8iUNlU}^WPCoptHT;{pKH3gt z?`D2|#7(Bkqg6FXj^MF<;RByG&OwFExO~6rj-aay{jmImm3d6Snx=DE0QG15*3o_D zKg9p-ZqWU9*6b4GS2Ky)lT25W34vwbKyalDhnbine5)t?LEH2P8L|p%M7h;XX2sv4 z?VINy=W7o90ZlZqz=O6ta$jX;`r%$^jJ2Q6v@4ze+w*Uoe{?$jDB!2ARVMI4_uNh) z+MEJ9;K4kRheg!E16tO8QS2PJhB;AD-Zn_LLmIS2(N(Sc;ehi63DZ19ZZ!!S64R6O{<=J_dgO(sK7*eFr+?)>`>S!v|FLeu%?iBHWN# zIyuoY8fX!g+y!#Q(4%f7XUkCXk|?(XCylx_#7(L$qIHh~SnLb0D~qhf8$U zv6C0eVAB{!+mRCWQUSr%9fzQyS5|R0*M|F<3bBRhNR=-1aQk+?G5hYu=_GuCqzb};r1rew^~+GR*~%xO&;3) z!$uvZ&tg(F$&X236~PLT<^^NN81P7zq>9%SwO=sy88P$HoE1TjJHK&bsa^Prtw+rg z`o44C4F1~@V{G1WA^xFsS7+MM`!AQB;S%3RAyuBvIJ^kuAwOM)prhmXYvtvgxfQz? zqUGVIC#zNN?vp;92r$kWbaP!(wS^>m+-Q7qVcg46!nJ9vyHK6kI3?sWmFb&h z17>zw3jDg8A)%@)y>-T8<>lo7p7d{E zN?!2Ws3Mk47~L(Oe>>elAWR}a#OvEs90+3EIH(Rf%~A3PFe!0DEoJa3Unk-ML_f@5 zUMIxr0?L)v^~sP9>&Mg1rNqgAEErr?wi$E(APa|mzVZ{ZodBZbI)4JtD59SESV3QC-nsLL))6-bEiTdwO!MDhMf)Nz2F;1SHSF%v?ewQQ_G*|My`BmP>?ZMQq< zy3Yg@Q~H%8Pnj}EXoHE47@S*e;B!Ioq8zP5`P{QJ8h+8#DMw56^pSvcjh~H+BJe4M~b| z{?sp|fPuyk8@-7GlfM3|lDQu3s1i2XsAC8{Z+IZoDCHF8TdGv)U8tTMK6@okdUCI2R%%lj7PK5AI@KvTHpBi<@|x$^c!vT zAW%aMK3|orcormH>H9w}03)cCEM2kIXCozg6`}o9JeEF-5w+?6*6P79YWcY10=V^- zBA1Do@Id$s0=f4+Sk6{u+lgS36-Y?AeZ_MWKl2qdIYnY0o!dSFS+zl*nN4q^?|DK= zAYg@$X>M*NcGCTkyH=hS?;HnnCfW`gh^t*Z0ua!KFR)tHtn=CZ_koZu0ib{fBzl+TWnc>#7Or5H6YL;t86*A$SwGcTfFoJLX|;6wTC18UM4vk zjiKaN#6z>ZDZFZ;Ivo>6$v?2JP&t_vs@lT1gRMU6E@E- z7>I%Z?*sT;aJUvSMg&%TmDBdy6Ado3J!Pz{nozWKFz#*9H|tBx<}pGfP!HSdJhe}u zp(ZgOqkf;)A8i?=p$}TOw-IZWuD@D0rtWfyxTwTOW+4r}lkhIms!M5lEU~T8s78}S zJ6Q|O^JSJ9FiHK1;oz-e@RjOJO!kTFU~DDHjdGIeVEY}h!|{Ab|CzU^_gG-1^xF1O zOAw)z#Q*ENEqHAT@ik@U=l2gQWLgrhxe5*!2Y0^oGA3MCf ze~342T|fJ<3}TpPAi>yqur0N%N>@+CS?_|WS$wXime8K&wy7kfWkx!`GS6h|Zv)Y{Zy^p*JeVFqN-Z=wcFp)#1tJh?AyE1TB zc`)$w;@nKn?Z$jYBPmHrE!#*ZybCuA34QvdpJl&pn@UQqzPF>KMoFc!S5{WSzdyc7 zo^+GbVPqNQ`nHVSwh&%&-9?07ocm+dZROqcr36~EmhhFEkH5WE@<-{RGe_BeFkNnY zC}o{fKpIT5C}Sk~ksxWct%kv9$f&@d5m5H{FHjZKtP_%NO$(iDmh%qorKLr{+H3E2 zrs3iAjkMaYkUaG$8d)!wZ>Y)JayWHNFU z24-oy+Xe>xFz(M<$Pctvg3a7GSU(OFzBoR6T$GpMtujOVJ+6h+ZE8iXIO3Iv8^ouI zzEUEOy8d%Sx<9t_8E8$5S-k5x6ADT+_L!@GhoV4c zw<+M7vcCUz0i2eO)2>^{%!%9rQYd0W6)qN3`l(Iy0I)GIf&6}P{^x~bj_)clRdMNs z^4YX&wbIX`WSKdgM=ie^0U{9K?CdPg_MXVVQcf8=J3X-id;2NSRB52m099&{xP!lw zyEwITYsm$RqYnf_1_5lmiZXRz zAtA0Ng*LWRqZ_)W!>acCHT2f_LEFm8gGb_)W*MmU6kDO^W7~pBc+sY`)V;R#gE3TB zgEv>Ae`^jd1Zu0^O07~6sO(b>2{jK7C8SRx6t}bNNJmFcl^aG%CO|+GV0NEO1f5Pu zN=kyAc7`fSMyE*ldMDQ1g?Bkv?q3?3Y7cYXq$e{YS?Lfh9I7u;`h@R;jFXHh~hYDhFA8mXd#X?&MyJ6AT{ zyME6&m`}r_t+E(FpM{ikz(^MsaBs7+G#R4@h0S%o(@=HLpj8>A7%AD*RN$CHhiK&O zD*^i*D`5!j5h)5RvZ8R2_eB75tiqTk16)yxXf?j@aErX}6oBb2s#ZOFjrUvTgqb*K zYh~i-nrX3uOFNI{jDHG|Wy>@X1L46^;ppuAlQ;;1ijYxE|F>RIeHLN$C*l*=A+S{1 z))4a>?gvH37u-V>*0rrEZQHEMf;%UX;UUAD{trNo%^JL-& z)HZu?Nw~|cw?%1Y=AK5BxnX+Y(0NK3r_8r9{GU;ztF4Uu?%!v>e{X9&5?Tg+l}L&l zh8F8Nx89`On`_e2j^s4u+17n;%B3Hq-^s9=^kUGva#iNw70mKdy~kBKSP)U%O#eS3*wN^>doo5&}GxQ{K1stV=~ zW`MP=l7*$oZ;eO$1_NA$*@i52`svNSCj!rS5Q?m75raCdyyrZR zqPR|oe*Th!H}{DAG2;E3$gu$Y<5v%h!A>j-5c7Q{|HJ*0Q{cxGKqLswhx90VuJ8Z$ zcHcUW-V&NW9Q|JUvQ2VoZ_c2};Pw_fseb=BHErwU~DP+$(bsONvi^GQfD+6lm62IA`0$k0n!Zb)nRytpWm zpiL8vMD8x9+DS7Hr{Eg%xgJusnBRp$KIBv0Yf`NDO)=CNkSBkZFo{OLfM|J2aWNK_ z*@ya!GU4&cKL!WHSRAye8F(dqw~ZLky21#01|AOakc_>2(Y}(`R7sHG<`hGyhH(g= zlm4M!zK9@4m1i3)$sFR_7KhM<5BN%|^GRlZf4%u2&I?l0m?m^JW7v4=P3XZ+IScQh z-QgXI#JQ7e(=zwSD(N~`cr)agR1Ke^phHpz`z&yM3%}NP5`5$l*dJ7Kg-d^16q6rym0(}+oc$!nHxo2=erpka2OPq=upv&UW{17#c! z%TOTblQjx(}IWM2#`x5K3+BtPYe_-7X|y*eWYfC z29?U4MfqzEUT2m2hyHrhH@h(#kW3jXMFJ`4bB0OtdmX1-a#z{0<8hgBY zfsZNT)9#SSxQIrNA=V%leWOaknJgli%0=`}B9mi0Xp@4Sp7f0mKfXXBfRNGatx}>0 zj%!AFKeTx*)C(t96Nw?kUMV%_fWXjF>T1uzv4kw%>+|wz5;gZ^HKEi4{%Yovbv}=M z+j401ATXclKfHJxc?sEFml8Pg4s+rc)Xsw>_BVWKN=@YxISw%B?CGvI#MaO)J`wn^&_j^ zawe+<13@NdG=R%aRL|?IGG+$~@&oKKvFq+PWy`m@p)6+0O+1laf<<)a$>fRm<9geB@F{WB-jZVwr zM`f0{CGo3&oz-pKTHLwm3?0TN-$4y5P9i4q9=Lj|ojtmIm}U5$03DV=s+5k;q(}vA zA3^2T%1cb+a0T8V-Ac>&DLECDYX4|rs7Hsq%qO;u$vFBeq*4cy{(Risob)6gE^Jqv z{Tp}hR{9G{?q%{Fp^w2@_yTK;i+l^SohWQ(CR(M_V{b3kg5^q>cN`Kfqo_P`nFb%2 z+v=K$!FZL3zL(PLP5l~TWQY|2fi!yIR}+`P2uOB;>X+n2DXV?o8v0cX1M{tu^7tag{yv|VLd?` zO@SLj`RX7I%Yz7t^+N^Pfq^$*EWE^s9|OhaNlNL>V*fW@>!UxRz^au%00Hl}>UI}6 z-ijstMV`N1`c!zfa*^`r-^SaS3SP7P0zoCaos)$hUf;2@&!$CDoAG4kDNvJ7;%~bJ=>HNSVm&#k#_yn&)T>> znK46^vB19pjo&HwaPq)T_g6);DODn?mT#EXc_8%d{bCL^qqk%rXs1~~?jkRtqo z)$BWIpPabcOFclWS67yG6nw8+3yI`>{{q6zp@AL+2~Y6v>QIL%FBy7;?dQ9M%~n@v z;t=HmNF+^Tx}7qf(LqA7` zR-1bn29MB?ymZSxRtKH&g*EUY;P2QZwXzyu94TB_G$xPrneu5iZ`RRiu&ezZIr=U6 z>sIweAQ2+5)oJFHkM5l0hDdl0x@L3zH8(f+15W-yfUl0&LxXJ}&UgK%o7xVlfTyxU6NrMvJo9*P{&(WP+d6;`h>m?5 zRZVR`#B-}s&jZv@j?edKVq!J%(m4kB_p(8Iva;E)_5jby3&{V7;K1Rpng5Cu^ttWX zEU>|OnvaNH3T#s-H><4<@CyPf+*^u^ZI@|$gno3~{lQmXg0Uz3Et|e$8WW_-fZm$k8qJ&tJ!q7A{aWvxf$=KNIGhR2`rYOQ~ zwb5xu+b`Ml8j5jEI|=_@@Gz(yiX>(XSBl1H8^vl3?85RJdARk*gjcBRPYTDmFnC`Y zoqj_cp`A3b#Mw~7>-A}@FF}Xg7D@QM(MT`wsfs4JBr*}t9$a=@hopGgR@s}XevKjx zc~jn2-Vc9D#2`}Zl@bdPAHWh`YEaD_kHmH;uS}ileO3O`xZoq zzV_%egZ*g&j87(qpiu-UTKaEI=R zJ9#p=c8?Kt;IeZJXRX$|zh;`wf99V3^vK!E0#H!A^)B(aeKus5+_Ox4=dPX(U!@=Z z1VNqs1JqAqS}X?n!jp)GpO-2OTsc|ZEj`jRWuTSr#Fc|0vJDTtQvQ&QA zFj6xA+UU*m`SP5n_Mq|1s7X6R5ckMET;A~ zU#f@j-a{)rUNGUNcB8^WH3Ln0NS~)X_Ct8^>|aBd0ueaW36GSDAB6Dyh|DOL{`j_G z!`!vVB;1qxucoY^zw8Eo-_f8EL3;4kpMQNy2$3STsgs{pFL!*HVnVO@uVwE==9SEw z;)Op$Rj@ylB8EKLZV!a4+gEPhU-Y$^8HjM7dDR1jTo-U4+3tul)*D0aoq`5+F?Np} ziqDJeTps$MfrN|n)%`rM%VI*0ZP4`=kb^Mpy@Y$&MrcW`0%TIgutOV53>1cU?%Pwd z#$dJ%PLda$4DY@pR*2c9s82-6O^esBrvvT}#E*&j5*yX+XA7Wmz*!LReiwUtdp~iZ z&WP0k1m^z$Z}@By9HIehr|5)wxf z#hVc)D6FuJ;rcZ6L5av8*4oyld;q|8JEgxGoQn-EiM`rZc zWx@a?-J=pjG?c|rdD8|!hx`!B*w&+j>~!wrqc%}pveXpF^Tq;fnYFUSI*1nOAn1%> zWB7T3ENhyXSoq^~#K}dtK^&UQc;TVyndNs3Sc({aAe4Tef z_EX>eLN0Vl#Ak(=P^=#_y&5-jDtQSUM`-*tX~GggU8c2K#Ar}R3Yk9%461oo_py_Q zX+R!ss26UJ9XN3srlDsb<)&xwy?rZuwl*7u<||3jE$^$_Jix2>-J3WVrQRI<$x}*v zr-&94-hh=Tcyp1e#e4f~3TR}&01fzMLAOr#N6z)cj{z->$VUgv+4l9L6XJb^c&F~4 zfnbF)dAQ5oee)3DcYGjGb@QBFl9wCO*QR}^I$b0kALBib_{@i6qDl4cM|mCza06sUcYFOISezjoS#W& zN08-VRSqq)$QrJ;JVZ$Me;yj5)z)j}mWlo!7huN(N};KI_&oS=wJ^THgcnkHz)R~d zW7fvVoRY{XL6+1ZMymS{hYi zr~?!<&TW{sZ=o0e<0Kk|_2eEC93OxlbVMm|J_;gcaWCK759TNliL=rZmV1oKJB$EG zbhX|EDw_YkE{dxKD3}L;u4XV`%fq7S;py)v<$pku@Mxs!833;P0Skv{5J!ue2iLVN zwbK-%oSSU>kqYmM&sMu0otA#5ni*+qKij*?^|th<44%=S9K!so@q5Qu*g|ehs8d9IE*(ZM-5~A+s@T>GoP&1n9R3~huM7?8sEyqmxX3vW%PLA5S5g^2h)GE zr5j}sjD}k!4oK#n0b0g#vk^Xe?667)B24IxE;`FY!_05H+yU8<`ASk+95&6Cgjkik zh!;`po-c>u8CGOCcjsA8siDzd5RCOlx7xL&N75PSGgf?s9uLTG)6i!``&!<`RpyUh zf^jBmlXeyeVK5k0)}{eM|24Sc-A=pz;>p8$!Ud(4_ z>oM?IsGxV?_+xz((`C|Yu_7reh!;1#0&O}F$?WC%)R!{>^e@EWl>DXn^OYHY$zO_N z;aXbFs}$h*^}J7KYBEaj7ZX8@7Y=_xc`SK?e-1t2G0%&v+-hq#tFoQbcWkXi^pxa% zzI^~LTVZvgvU3pK(x}$66|1uq5V3U{ru-)ij6TEwx95Oz)$(&T3$&%ZaK<}tH|zS+ zkAYchH@Lt)yC>^-lgkRwWrzeXFj&8u-!B1XITv3I^c#%T4=8M~2JNs$eOkQu;CCod zolx^IA>%&qV-f`O_S&tZW=G|dsit|q4KSa3ZO;b~5Z$u?XX?Hh^zT3J0RS+wUyXp` zy8~jR$%2F8y#rzg zIXixRxm5}wrh}5!&_77%w|=9*nV?BXc>-*&)9ItJ9Vt|N(J{-%!e|FCLfoi=Y51>b zCyE1~r$4@REOs!XF9A(b6y4^&eo_W8X0jUSut3!SrC&85PP~BKmTGsR}R&H;7VbKteqIV7jc6|V<}IRkXNim}xFcYCi>SHeVag}ja9T{D_o68BpijD5Y* z%e7Gka1s&-otibU|%XXRL9j5M-nLxnqwuzVs3UvL63&@L3j38dtf2t)>l zb2qlhZ>+Q*tCM;0QG8l-lHig@YT9EC?`S4P$HG=wSSuc^WUYPUY$7~`8)|o5ceu%z zleNTVx1ZFHz0Y(>18eYx9`2Z!RhZ(kmL<&g7Qb-q<_i`Umd1xyu9`nwf1(*oZ;-HV zU(G`ov=Sm?emT>n^|C%AB-GN}BRAQ`4_(vsy0@L2r-@ll*PCYh#ORoojL<$oU+7Ib zbyt$!fe!-B&jO!gCNv}c-8o4V9Qs|UJk+k*jUtWgw!IVSYQ1B($Hj{d=x|t3G+2z8ec?j5nk7mvKPM+}#wD6B5c&2Q85u-n)0xV| z|H~WbvAJYK-wZn4JkSIdr4ZF4M1$(dxUCOFKL>y?7^CmXucQ!(*l)obx@iM?;Rw-V zoya;BzU$k&HP>?W=xw+AyVO7sWOzzcsDK~DmI@3CeZeoyDVc-Q-RRGS(UF+*HyTaP zk0XPAHWn~K+~i9*ub7{fzG@q7u=9?jWrO=f<)t&FLwlfCaG~0$asyTJYL-h7a^pn) zcn!w7+Ym4jOB#`Bpx`4emPknV=JRxmNy9^3TDLqsrQ_kFAhwMz9LK!@1sH^ztub!T zSxF3>>Re+wQJpaq7(fV-c}Is#u~85I!Y&oThOY0k#%4T9z{Y85K?L9^C`4GcB)D7z z67enEjL+j8pjtZH<BjmM4~E_Y9cArZVdAF|bg&`DU*$!qWy1 zF0r~a9PM%z*2ii&ss39)Okvi=UZeX+rHM+gs{Ul*~t6h+(xYLQfnARGD z)I$2Uy10};vcgO=VC|aJeO$~Ai7iu_*62g`UDna3FX|db4;u!S8%9b`nI?LhVM*`F z)cR|AE|8L{sO4AypJmIk+Z9X0P#RswaTcL3~o53!b!GwZD;QMwbAUi(Jm;SOTkCh*E9eI8YxL)N5dBarCTQibnp5s= zlk4D5a>(x=jE-sB?;L+7T=gVk4%`W!lgLWgNVHHSPxMngQ_5%EnL3r- zoYWdsMl=aj^pJS&a=z5^Y;~+kLO>bl9|4QOD8rj?-=Dv}I_F%#9vy*oYW{rz5`y)( ze4T+K_$Y@75*?i}MS~rp;p;IML(N=5{m44n%<}G#BrV}{ff90>_0I44JG4ax8IBc+ zP-~MXsef4rO@p@GaaWVnfWH#3?8G_a*!pRbD2XEV3U`IbW@sDui3*2oYT13wA~WPK zk`XS9(c~cq^emNU^N9Kod9M=fcq9krWuo%7;%(1 z$4W(D@<&JD@`uYew%DNaJ+goa2l-1C99uW&j02;xYs1FOaVQ{It6#SJ8kv{uDlv zIvdjJ==_9t+gG%RW1i!yNmkm9Q-MddZRNPCN}&VK+&wU!Jk_wdHO|Nse{t0LPiOY+ zBpC=rcd7Lx_$q#JzuAAsxtLUmGK9db{o00Fy2FU~#y=DnSRCGaAx z7tm3So`*Il(n7OJO9k`G0=E|W(>}tAtSgVZo;*K@wh;cg_ZU}_kF#;RMQE0_E;zZ} zrLX2S@k#a!r5=PSn$(`Za3rmtSOWnC-{jd>U@TMcZMYmG?8wQ2K*eI0S7Znoc_4dVm~)c zJe(j^P5(S>^Y_5SzO;93REOu0R}*he#$oTf)vxLlODC^B__(u5)BIWE@NuG+RR!-# zVVLXk3sVRs*|%;oHQyB#c%KbTlmU6k)`*W^3PTL0gi1N-LHxP?D|ZOyCjbHK_{3Qm zb0a;6|K7ZwR;K!_-}+C>Bgx#!n%oyR;z9EXvU2r~I)^FC#HigD&T&vg9dHyv#f24H zz;l9m_n~2olHVI)&B83&vPD(s2KGV40lC|pW{TEMWyI**XOZu}*gt*BrrLrzeUWj9=uYBjTq8)J|JC2`U}1>0KiR03N+YPlDPb6=wxlI28V!cG08J9>J08bAl0Y^!Jmu6?3l5c>t#$({8+ z2o(xp)inY65vkv$Z5*($IZr+=7|_DCo~+J)yYJo`oZOY$?V=dG7b(*M?st~R}wBYlAHd?-9q7O9MWAWk$u+dDa9=NZQgov| zOj*$^4*;I-sn?0?2@2KNeSX1M#rugYG)IkV!`U4)$Rcb)2nU2Yn4j6stPQi*7O%KrIh-D}#4ZV>jD?G5@t%u?#0t(FKG%;T#MO50c zca3Pc<+4FvkTbdp;%*U0uyU_6^Z{sa1?g;4al^ML^r1F_U#Rv36xw)4g2SKxnp@JG z;0|1j_02A3YE&R9pZxNPU?IKi7 zNI=3GX~q|qA%Op62m%ehyV>X_V366{T1v^gH=Qy_o&JhB5JYY8eRss3=sutXAMy|q zLdgB;Cd&7Y&p;wI^7?#DVsuyuj6ck>^rYCLhFCywXc8tg#!(QelFvO}Q^v1%`f&!o zk5fW7Ze8Jd5@6m{Vj>x7xjb0}6y1M6g^Y~)?JXPwkCem&jk_OEN)!Nq<*l6uc%oMv z$QN?!ygbpUNt3?%Asy1kWBEmjUh*=?H(R>@{fFG=6))F+9M&*h)gAZ;d9jX=N*vbr zapcrCAB+!4{~Jqf3OP9_2rh1!gu4Hv;gk7_M5Er$P0^=FDG5&vlJwJK=A(YX3QIz% zW%US<(a;46dk~s9lQ$Of@s?%27zL2UkjFh-Ykv35$%^-B*d^)1BJ+L1(3^_$oX>K= z)|-ub>_?v$-w?=P>!iXm^yI|pdQ$8j_k!b+>K{xOiHA?7!5n&MsIjc9%G{CG)@ezY z%^`MF6vns0TcED{oCPThC9h=sq&vO#v8;>}ai>bRx(xV(qiuKkOu4e9!zMh&IJ}F= zcu3Y2C|`T}HsR=(j#?t&@a4l@^B9#eeX2Udv*=A^Mv~Y8B$V-v!Ni@XQuoBS3~cvx z6p0$hJxGZMj}+$%9LwVqG0a(aX1>nb#7Is^P4f--lbfpgOHSNSJ*C)Q`=dG7e}{#3 z7sHvt7Nz|sNMYXli8{OQdyNQ<538b~L-zSj0YIhcaBbHvJbz=s>40b|F`7^So+wMr z@|RyylvD;y49Avk41ZN;Jg;3<8b~6Z7bAx1l;`Bi(f#L zh$aJ)m_*2BDvq;|vs8uZh~`~MW^@=7MwsceiNcb}X(~95iStW= zKV7^Usa#|V9({XomAi-qxEB!TkP(9uH7zwEJMo%835WOhB(9S1j3YKI&2D}enQ*D0 zyqpV%KRr_2dy+iq^hP5z!^zIJ;cxI(?n2J!wAT;$kbS$G>#sF(UGJ!yy(Y&Ld=6RK zukIFd58rXTZ{}_#|LlMmtny=Cmt znDNJ+23}xp!m>g0*#`fWxpG#u#CdGDy^zH7HJLbJP|DN0Hz(A0voY(rR02A3#PXB$ z@3G>W>Sio(0Sy%>8LOkZhTd%Q&@-T9{B>qgJXKba%|X`O7t;S$BU!_^LwqkhP_}Gn z5DeP9lsCs4_*Z~*PEeO)an1?QjJA12aq)LRIs^!fUjZbO6VLk*RUPLzB`KT^fSQW} zneu?=04Pue@}TE}FHV5!@=q&PK!}9D)t~%3!9bSEQj7N%Af`*i#|FWg(||k*3jh}t z=m!#qq5;fcmLOE}-_zokofM3wIXR4v>rSU$ZJb&C%=y>noX-Bfig#mhx=~5L^W>#e z5g>iKI=$cdPx{HxpX(zNDm{mvkCA`Yv>cv)3O65ufePu9=Xzxfpkj%4xHEDHGTBMI zuO9`uGd-L)nZ3ALdFxQQztVnc&(3NO7l(|GC4vNQLL-!)pTdigAHXDWg|kUXWuSYv zN`F6U&_-%=aWY5MLeRZIrLCgREO=Ie2#z^pmO`zyiFyg(WMtUW_O_tPt$>iu)yp3y z%XOrLN?qw>CacoyL4JXOr=~#~TBi;#tswA$t|he0&z~{;#0FlB50&7yRDS&a?u*-D zM5GIi1U?p3CmlYzYNalr7?0O3B$Alfh2@+62^iA3Ftl&{eX^zQ+3k7=f#gRtuxnQFvIE z>1}90_N^-~f^F^BpDczMI7tPvNAs@mfwGfb`P2^TaudlIaRSLVuZc9-C20WP0Phdx6`9?m+Fm zT^v3I(5?M=hVqWn zFa}_{{`27@wyhJu$eMwIq5oowqF(-u0#YZ$H4`j@XS8C4SbmvU7Lb0 znqFS~J@uRt{ln_Fes%F?)~I-v>F;6GKpf(~;jHT` zd8XihdYu<%AO1#1+_B()i}gls$wXX-J6mPGrl+*#S5w&lR~MRV-xOz ztSz+5{`QaAx4$naQ${~K)@r#AE#=OohUEeFb5^oFd#}I0ns_yHI4x#RgPXXKrWM3w z9qwghGio&v@}zk)r*a!C7wtO7X0<1W*#R)$Mka?fS?A0M@Y>)gja+v9S9E@>`9s}z zS)a$Zj)H)X>oZb@zb?DBERtS~>K5H}kt>_XO~@<%&Bd^}8o~P>{v714lOXk`M%)iM z84pecm1av-c6E)oWEITofs#Jf)IwfXMJhQ~UbbVd;uLRgwkQaaRV1$?g@HdGn`T>7 zws6Wwt+l}3k3lL--(G@7)%HJ0Cl5t6n)B+ZDl}*|L8!av9T@SR=WF_w3*bVY0oz*@LLnNfcZFqAKVARl}b1>SB5 z7%JX=XixHU!~Fn!6adh=`(Ejdo4fn!^B=GAwnL(oT_95HB@k_Kj)zE`opc)FJH$ zHUQH@*zaKOf7o}u@pIAgYV*_eS)!5jwaJ^{Ve0-#FU@U+4BmqabHP+92$8Nm)Yi*T z-rLJs!IMG;494ZMnd}d>KXnWjvUzT_OMyfq2gCJeKX?e{L^?(gqWsSPlx{eQf4pYO zC4d}C==h)*KMLkZG!~TMNHD&pACs7E*y)(%!dYYVWDL)iA=%zlC)s1fQ<44|vl|cD^VvnFE$eZHAc$PXXwgwl4pG#2 zmO2Lm5p`B30EY@pOOl1nYGNDMh|T3P)Y;|Bk0iorgcW>{k(W(No@V( zc0d$GV_hYx!*ww9*(TVb!!}1@)%q5Ff>}E*@}t3}^zk6O|)*u$F zRN>uNBdSgu5n;D{P`3DOZEX{0Im$`=PUAvkGevjRgsZ4L!_2v>FK78}? z-;Y%Qx5J}#SHM(zs3(1_x3$n&hw|HNn8p7a%=pp)#dR!xUrOAJJ4dZkh^GU0s-!m` zNL%x(YtsWptr2;{MatN*^*N>MbuFeNDIlC}B5Z?h{`%z z2M$&;XMc(mHDZ>5BudaLgl1BQ65o6%2QM`<;X)IKqDSGO!a_7X6>K*CdXtmIn#CzE zI3=nUKlk>QJ$;!?N`M7y1oH%pUkK#;L=8jlLTD2hT`q}ucoiB#?>7qyz(q97b{zF^ z4d*6K`DICr0C@vpSk|B=0M9Qorhqj>H3T9o3Qctqs`qtaN!1w)n z?QA{Gi zx8gM2=t|O61fv6iZ-=)S2OF*#z;=5nC~G5 zAklM67Y3xmo+7@J+b_y9cn#Gy7u@a34LIVdOfv2c&0IX)YqvQV4}^?eX}fcv%7hOt#InJG=+R$7KzR( z@{C9_Krb=Q+SDg`gj}X${yK^KmY%Z!@jL{|!FBOe+|t z$Mg$@mw$u+0crp62lLyfD8K;Zj}HqB`OLTt1Oj@G3C}J8w;l;%D+2Xs)cW=dBT+K;Y-P0Ne z`;Z)Fp9!VuR?wS4-Qc7w(E=GkD&n-HpA}`qsKf|_XzC=$ca~+;-x7cV+8HUpxo|;O z4n12zf4#-UMZ|O-F@3Z#1QQLa$mm%()wwMObOx0=Ysewjewv<>BL}}zydEq{W#6(d zLUF6#xXD6Bin6hV?gv@ZL19QT-SjL-*O>D^Qt0xS1@_r%#0w(vnP;RWtjC5`$YX+XR^!ta`C{TbV7U7u|3%uOgh~=&K2b-4o{&V6nMNXxJ?9B_n{VPhBGw z1PKF=!6LQQr-dTd`XIT{HR~T68m3*{A88%%X&q2`yFvw<0^dy^5z(6F5#+W&;Z)^; zmBhahU${V|9R5eaN|c!US_aD<_sU*54d`gZq~<$8vM@Tx-Y(<6>qC=h&R#he1r^3{ zkX%ZMxMVJ-#EF&8mYoE|svk8!K0b5eC(x8a@!+w~%8fg&2QE#npVZw<)w>yPGeSP= zsyt0S&vzR~i?hklfP8sh=>bvAE%u1p@OVhR{?3@Tfc@sa@8aX*;34sTdfQwZW^(5< zL2zCKL{bKKzP!Aw0(PAg(x@}vhbOX#XfV8+`fr$A)oW{OlRov9K8^SP$kP88Sk=Fk ze+lz8DNOk0pTIZkE+4k{KeDak) zz>84!OW<_2idw5yiFv)HQD;oJ*0sbt(pQDP5eUMZ+h4jH z9T9XTl15y#+U^TQIrO2-WapL(8Zlk6b7RxAk-HFEPdj^A0aBPOoDL=zE+Vq$Baz5V zVW?`tM+Y86hEgaC8vy`cMT%}75h4~$t%k{SOz7+S0z0)(88REpsH94fK^RhZIkGj%FY zP~Aq%)lOnVmQ+bh24I66$tsJD@*g0~@QnN98MebTjCcSfDj%IQ{BWy zRM*$!Xcoc4UgT|Yi-Gb zCAK|E8)rR!V*ZSa(2dB;W1M=VTYXZz^ zK(hWRlXS?xt}r6JycP)AfBDCG>90}9hSBEH8J_ss0V2+2w}?&n((zs-ybqV4Yc}IS z`6D&AkB5qgp!pHY6MV?8NLG&XfEO?bSORw@{?%4k7Y+F#iZ_FM4m(YCR%NmaQiERX zUf(@3r1X7-^7`Hp(nu}sf3g3*D&zi~#q}@j#gmt(&ryoG2+pUe5N&waR(2jwQ40;C zpvV;-g;vO7FJ)I!^6?RZ3d!i{5qWsatwS>IzLK{G-AQY7<(TAxhRyHq@2{*_Du84@ zwKifPh>EN$?y>ogGO+KTg+i;%frjp>7#mg8PAU;aGAA=6%l3Q3DyYT(m?$&HuN$P>fM_CESPD_uw%&Bk=Mqk2Ig#wj86+P>@LdI|)Sk;%@N zhQ-Rqg(^?au)5&1%ik}5oN0l`krUyz{u&B?DLmSmBOG} zen=8ODR<#IpN1{9I42>53`$n#LA_YQq0X zORgR^{q^XscfBu94g+ZmAV7%nUX8KQhZqnu!NayHv)xRy1d!qESYPlGGyQ*$ulGha zVAFqjj8B(fn)PQ#m&0Gmnh_}TA~N1fY6S@;mi!-F3s zFE>u3dfo&rJ%xYYWV&9fIYd8B+1Pw}xDs?dlI*vCKF}xi&hqgkTa?y{{~t=rqnX3e zIGL@uR=z!6!@Bvn3^SWwDSC|^#!bwhAx+@ySWHWh&LhKG_G#@}te)q?28z43 zaa8ZiQgt&18#1_?0j+a}Bq0*jbNt?~T9MOwvs<g!i~E62jEt zCRGxiTRV`FqmEk2#+sUR3UV|UxS_zE_xxA*hZR_>ob0HdSWo6UXuO z2XYBr{b;02iZ|Wn_jn9A9s?smjf+QM$eoWPq&ia=!i8#JtQCg*N=Cw>ii*7zhE2FE zdQ_cbQ`4d^BfP&+Wh7i#+59}@e?y*LsxAR_&uE#HB}o zB)ukcK6x-+_i_0&mPEV(S-6070mDck^r-oKwk(ubW6A$>k9!<>wTEnXVoPnGQz+-+D%y9gn%Kf4- z%F1g<0rPjLQ_Sz*+rw8I@}_MAUpcOS_XfYS3i&oUa{VWRm-TO!^i^G(^kobF;5n?8 z!;jlj7(4}|>*ngs>Z;Ukj@Y;S(XPh2DeNE!CFS3#l2F>@@2>?&iN?c3AYzeg`)pfD zBdnU6i;`mn0!APuVgt5gI)gDTqHvZQRafNxOkd@>*!+uq=Wc>Z zNaCMW7a1RyO^x5}+4dQR$C${0;cj%c>K0m~!Bz~Q()hscCr3KGD3lyh234c6KFkP; zNjv|FaJJdhr2$QiE~{~+PELhn3&#<_y+BlXqi*a1xwf+$0}C%)K-x_W*6mxu3{jM) z+5u5nEPK*{zepc^J3;u@hCDVHUFuH@%l~D+P;ss~h6L*7Wie!+i|mAG;YMo5%t&SX zuZ<5U_gXT&!6qXoGv>ze%P&W37aCJ_>uF+YPIT|^D;fI+4=FSlbG00(tI6Did62Wm z$#vzUe7QymM%D9s>75sAGO6>+Sh$;ur;>x>G=#nPj1-<6TaS(Mykw1sF*nm17(s2I ztQDH||Mip|yBALuYZ+xC-c2+$HR-8R_P;mgO9%Y|hW$b4sIP1T` z9;IKi@u~KBl&^!t6yM-s15(Q&a&%a9v`WC-G3WrFtCa$8c&Rk`Q|B4tJkX+Rvd zAP2gJ9vI!9n|s3*9AZ#A2X# z>-kf#&N5VG%$RX7Lwth@s6vj`Rb@ME(Fz@xbvo4mLvw-Tvx>CMtbhaI?Alo-#zXwsHoyh#!Vy$~4!J`iDxljmmM_Lv@zpaFo$&fz51#u~Ahnq`oQ? z>ch;s^-V2ZcWG&-RFCnz5X*)xavh^;74w%Z(} zM;6s#S?8| zYT$=Zh@R%=N@VCxTq?GdD3b+d%*8c*cx;+xkMhN?%uene=c)|>$aDFIZG$<)=3U+U zK6VjTwbA5lm;YT{OqW2O@g>WHLk#J5#5>Jm0J{sn#rJnXv^*qDw0&Cx?{r}*P&%&% zHdnm(7GLcBVJ}(2V+{m)d;~Bp*FgEg_=Azcbjx$#nLK&<*W$h~F6+yCrr@DZ5Bwk} z0pv=r2}5r&%`aU!FyT%Ne?0Zn+?MW;rqxL(0PR+#71PB|j=TyVFj>@$0GFM_Zq%~6 znwaK`h{zTkw2dz%+|nfthk;|;iaWG`LDYE`li8AmZJl>&s5m%}#-QcF%IJ%wzGDu} z9y#PgXaq)Pee)Y-=PzIv0R*29g1>#Dh^Uoxeas~+hrI7U@e5Jz8}Ud-pxmMPE4Jgn zs5~{IF?q#z><%1R^_VkzzFoC1hd8|88XJU zXb2%i*{Xcg=k<|~_?taF>dfQiX+L}11;GhIrpt-fcI6&Ky>@qu2&DHcchpsh)K=Zr z`>N|?fIy_{+JbdQ;y{ySJ4VZ1)8d4LoB5}!FV)4=IVMFKQKt2yTaun%Yr%;3lJpr1 zWo8>G=T+VYdXwf}vjBBLt~;Sm%~Sd5p5m>%#$j<#?9>4vs5G-I^KQvz$AowkBmxYD zdOr=y+7?*d%}oEkUo=x`2<{p!>m$HClgQF!PYzc5^&Y=2{s*rq1~b|gIXwOm#GdTu z$^G)X)|)pR_WA~xKYfzdS3j3e8JmB7elDVz>wUsv4YP-ILnQ0;%KI#v1nc+C9ycZu z8wPip`nk+h6Mx?u(l<^rmF4q{8I=uhOU=Hh;t@CFDq7gmtuHaD+)_Y_M)^J~3}jj( z|7Bck+Lbs|;!7h6vxhzQu*T;PY%v8IkCTW!=KWU}0JLxTrtryMCoh49m=tbs`K?2$a5O(w-H(D{E^|VG z9zvo?um25~w zM=?`qf+y$WBAA(5Bjg+Opw>;sS4Ue1H{lvC85s~g)fz{RIxay^hygEKdbTcZDGs7G zu59mQG0S@=NVeJ47NV%gTrn4yfqNXVSCa<8)XdP|R*@o>Ga3tCZIg^8S7a%=O#jJm z@r_~=)M(0gQ6-NwqWvB-BnL*C$82WzZ#De)lKl6Xu7++NRbHQ4y`up6{~H@hK8^pt ztwaD~c=7xw`}+77FgRSkc|Ka$!t@~q`t!wnUH`b|Vq@<^2J;@lt>p5)?RE?QH?Apz zUDo3MZEgE}%Ii;D7eN&i!KWm)0uQ0DKD8Jbc<5(KEE6x_6D|5a0s zt8C_!YfG2Owc*b6@cKDOl=FQEEvibPujR@H0h-HneMNt}y9;;aui;`aFa~kF=a=im zNjB!QrY-#C##?W~UK^#MijYv+$7}OX`;jnW2JSeEW$qwQ7v#H-gk9pzk0pjf{ImVG zANIsS$Q?zdE+<5fZ@^~~oXg(@A}hWSy0Q5LW)Eg$yli{COwN2)+0{MaSPjH-rnn4s zR^|NTUiNxdjHkQ1`{^IIgwOqrsNUMNA)gs__P*B{1hB%YgVt<4u2+y(^JC@>R+Waa zg=tiF5a>sRBer z_?{A-=yVd9PD4Reoq6P8fpB%k80ynLRb~0}?(S|9Wq}MOTkprxXZY(q zB$2-dLJ6Ist67Q)aCzN?GvX_R)P)gHv{afdy6iW|kS2^;$UUXVs}eEzSMSIb8GL5u zZ}!IU_KwBQc~;oAK7E)Kfse@EfJYjc<7;mS8?h}`or7)b=CZZ%#pZ$(G05|4w&^Ym zI2_KvnC&c*GZ889{MY=6xiM<;Zk;k_&&6;|Fu8GzE7kl(PMLRIc5+&+c~x5MQ`J&i zNg5K=4gJvX6c|=rmgg<&!#{LYxi(-8QJlTP9~F@rlT#O^eh*Avk~%7Xvr!FJ4Q?l$ zHuo~!-Q%T;#YL6f=|~fNQu=@I-q+{-KEKuoK!feUhvU=0HA^5PxiQk||9drLUw9&D zUTweSsH$qG-&>F{r!GY|a}>2`wur2lmnoos*ySHMgschl|`un@|R6)<&di5*i zk@Z8bpttolb~LlQQF`-wds?0#ps>BULCRlLPyRblWuy}hfstKU$Ia_+TOS`Z%P$W- zK|E5(@U9|PUa=Yq&o$Z~LvX<-UPCtN$> zh$+P8S*9L`mKS)e-gQw0bDNB9(odp1ee#7fADr!ujA;21GJT}asb%+~-t?hSIc>A3 zYXY;(Gm3cQN1dOAHWSpg&#SZ?Exh*DEWMpmAEjj*7@(uTSs2P#Yofk#vd4(Y{NP#M zlM}-cHaLGG|AoxYTpD=~Q&Gz||9el4086GsCE*2)~RVmN) z-yG7O0p~gc`zwsAjsQ+D($Ew%T;Vd-2s}c-t%-^L#J#g$Cf#n%#*?h#a!*lFEjFYD zf*xW`D&PEOqNp|>C*NHpPydB68q(EZZ)3ww`YVjS8Os8qhRLjJ-*P}J6tn=9ZStWV ziweYq{HH%S;`jO95Iy57!&&E|n&StUld945Cd7$27iz>8>-8@&(%fs>QL1iE?tD-| zmgbiGuqnAEQZbDkIU79|d!Eli0Nh%xQv?Y|Yae(7@9u*XT){+DjEK)+OVL$!KxR9J7~%)?p&*sfZ>FLZ0*=8A&I#nBK$ zauPGRh@)SNmjtQ?w3wH1$M(|Cp!nd#sC`2VRkF|J!co{n5e^4VO~S5wuMi0Toic*)plAXcT+}< z+OKhGYkK8D*dhDzkB+zLgdl!IRBJkNG_HF4^WAXe{VurSC8bQ*{B-f;U8*ox9(bT* z@nVhiN+|t?ShY}@-|FRQ>C+o}#|vMHXGg z{(V~Q+%iiP5$U)yJ1^pZVNL~*I7HVR81~c(zf0E*SDqAdn>oF7YB7I!Qg1)0tBbpM zi?y9peio`sfVsh@pWy)7eCx$ms7wz3P9dDC1S&Gr#ToKU_n>82k)=_}&eK19kDMX| z^}e|Hmk+4*-|Z1}x;DHEO2EX@(+i6Lch_;C~Zu zF$H|F-}xFO-G(!gykmKeIjjf1!Y#!e`Wb}Nn>=xz+sw8z8Bu$5}82&jgDHeLc(r_jR|e(>ssCS z7$YY$s6;ujDub!)tl14LChtoJIXV5RBX=Rx_O`0C#fV`wvxZpv*NUpgwG6U#Buz4~ zkmaQrXh95_lk-GmYV&QAV%%8i34?$6VeY-(W%keNC`5cXe(;l8289<*nn+^N?1t+- zqhJ}3zUP#+L^q-fH5qG1~`$4pGbK?CGxd=4N`2C#%GQz)!L?ve*gf5g%F&##>Udy=U@4 zz}OCBj-`GNepT}6Iqf69cJKdb0k9F+t!$`(Dzl7{-FHp`s1We|d0T`A>Bz;w`Ulnd zedzNpTGd*sQYlyXpJkFn(R%N+YVu||bG6Ly@9pgePe0lI|0b;W9Wfc(05B&3%5MH} z{YObN^Udo)N?%wPqDu1Mo%C^{lEq=h^me$+9jWsq$R|8|;GITTxAEfp&%m zY#9;6i`s63Pyh`hRUo$>x!TwDTV_;&Y>&T`@cKRVz*GMSCuCE0w8#nr!DQ-$lgvg}*J!m)-*T9vWieBa%wnkYnDJFu5_ zjhVj|XQpDV9rT)}ZOClu8RKt}gb6Z-&YJL;Qp2sZ3|&E5yj54VLWZwoiT1jK{|pd- zoTw3@vm|hKb~(MuR3X4mgi5>c`S~xxb&)|b(8be9lmt7DLfu0yeEu_HKrPOHYOx9= zHVCBaI?OAGMKbreIhB>FrP95E$US2P^Q&MU=4Ey>eD?TdoT`gSaJ06UrSI+@tq0f* zSd5Usur`Yn!$VNU^zi#*1vAxE-{4sS+hMNL=&nV2+qi~kl~|`Qx~dknO#f%u1tu?? zRTZEah2o+7{a^E1*&V0eb}YX+GxpFUowJame``0GS~s4sCIONTnJpeR@MsF%$dA+2 z&LEt=%-gXojD3;13pmI^+}8=KN+zKCY1|T6>_tR9_>Yd`Pp~cSppqMPp~h{A(9&lF zAR$hLF%2Xr&u;(?A>;WC0*Q*Hs*&=I)~YW%?EkB1Edjtl9%@(;U16S#gK;WdsSL5c z@K!hAiz-d&U5=sil@D)KzP-kEjKb}g6|xU<8%RWWL6*sw`D-K2LX{v1 zXREpmEwJzws17;7%q^2pnVs9dD8>9%O#_5w!^~5}s^!`1sRy~4r?v#SIMLE~6>66J z+I0M<*|Bs6b@_&_$Oehl6`NoWeT^s1>Ux!8_ZPcds>5Fp9jVz_t05g;>ST8TD?S9dfQNHTP^Vc832*`_`etjWy0Cu|M~dm?6vh;4gS2M%)+qIFx_ufJTt#C(JIpMY-^@alad)PMGvc#I-JUD?uB8dN zLHcl!M3Zh3gb-%+yl%Skng3#Bn6eT@-< zO-ZXE3sX{(B&&OINHf_{C@X$#pvXR<(r*pwQn%JX>*kr4k08XSPU1p$K+G zWCI~63t<%#gD!H69L>S`NJw4VMCoz`1+{6An>1kSEmUQDb!&A|by#$Qnj?EMeSR6? zT)iuf_`?3H;Fkhd1$qz~`^qj(fP_@bbcrz}BH8tU5q-9vxOK7v?E zpe`v4M%%Pfb8kO2e~VF!U%L7eECK8}>G-5wIrb@Ocyd5;H+fVPpVLI%f5)bWkKljRNlXQq_yDA_SUX74{5jCe`DNYjO@2`}F(DzrcVsy^El8&= zXq9*$%7c*%*vzupd66{VodU~ZEr50W>85|Zf4!dUwwslE;!$15N%}kGPcb#v z`l5Ss8JZ}Jv6q`|#lSccsEKvWGUgTUjoV)lf7d?x zg((|O^@H*MeVJbhZ)7vUm7Gp0kNgUETJHnY{$L^t^3P8t>A;V1N|%&uJsQ*mYwN+K zW^u`-@7JJj=5^bz+uj}pTYW?X|9%t?V3@=bRzFs?l)Q^ zwpcxtSmcw%?Wawbu9b!C+Li9^)nkSw*s}q8Bary{AOvaGz>lYJS+EyTq0{2FeL{U8UlWd!IZE9ee1zj zLoc2sK+WMw)YW2xxld{^F9*t1u25e#ALK*HJG+(h*g|Gq%!*W zoXa%EF=yQ?s-ke}Z?;oA37UR1dQe@BD|mG}bkVkE*4M)I%;c!FTkXch%NzVrx~e(~ z{)9_X?QO=p8y%t^wrO*tO-;qgDi%eL3U^29{|^qbngYmuuh#KSHTiGKfTVo_uMS?l zH}D&;E&{q5&vO9XBtA6~s1hsX%Y&EKjU#*)U2O+X zMBL`y$ZR>sF1NFWSti_21zzs;XuDoOv%ve6uvyusKw)2NZ?Dp@h5Q>02g-kul8nTg zEWa88_*T|ZCNEDz09A1jka@mCf6Gwc^RO>!I)#j&{kdk#qdxDN?>AkN9ff$5S9(R1 zc^DMJf#y|ht@mjc9aenl!jDt+{19diwV6dWCNHhu4zvPw|(60L`q%qlune&(0v%7hJkZ`B3YHW9^=mK2B zXWLywG}PUGMKQ6$6}7;luS1K$WLDjOl{ky7g=96l?ij5uEZ`+VT5lhdZ4!6E&5l}# z8a2##$``lmbyFL21Y@zONdcV7avn;^x|_u%j+u>)V4!U;Ek$?e2578H-8n?lKyUSt zks7gRB~T5e*>XTHfa&EdOv%=ZvjJJs4iN}Wetui`;a^G0*AIrvWeDEV11$#Ww_a7t zE6I*+?d^>66(l?;3cILrK<2CX`n;&YoDWFqJICh^t^TH91mulvtHD5J7lXLhdUwb} z)CpeC*4BB^CXLtR;%hkZQ6T^m8u&3o2@kh8g}gOBn{okUmK}c%DH-nLANr2X&iA2% z%GQ?Jp(xqhKOFIl%aZK_oLgg*)uB8BQRE%F=X%AGV~U|NV0lZjQ786oa-{V)mqIL( zZaZz{{psq5!YSq?ECGFwb@)%+&ZaqGQGbm9gYwuqI@@OpYanD7`RFbk_&%ImxVV=X z1RxJvNmR|Vqp_vw-{E<#i$y2Pgg;xqE5!mKbg8x)i-uMJ974>BMjZElT2KCAy)5oM zoKR#|H{_xyhcXP70_T>|>Xeo`nJ5hogV-TaY!AC6zV#$1Pnsmwx>k4}AJGJ~vf)Z!+^m z*atl$xUOy_LPxk_^$xTuB!9^MH$GDA=a=`feQ^IA7FR~GM~!I^;hf{u%Z2L4*_yhQ09 z6cYmue0K{VhA-e-1-#J16i8qV%Zx%|N zT??QZdkZLQJAiKCg_V_#jWn|mKp5-)96 z-BY`>!?2EeZwa%o1x~up7EoVH(&t3bbCrf1nhd0f55XKoYntdXwsOX^rNp+*=M!ZI zN$d^Z*JRMh8nPNk#<)J{5{+v2f?toi@UzOYqCSJ2V+++Pe*1PuM%y?F?GZQJzAjh) zC>n5bcYRq_2q%`6yV>)ROl!J73mD_&6=P(IB1a`j{fZ{Z#7ek85P`MUUL{{hMxsb}B4uAj@j8q=oks()hC!y_Skf5L3Qw>Bx1A6o1ZW-lPlAAR{*r zVJ75Mwk`{~MMbS`L9Oh|xPLQ$KZ^XvSat}M;_wO?d~+4vI|Zl$Yw<^BwMTpH5_kuI z*o{Mc^3Y}x-WJNu zf5uatXX{x`lWIse%VQ0#E{!&aT5;x^6Kqh0bLvysdJ)~@pGY1E*qb%=9v28a_J|NS5P(7`zwA@fj1ievA2M94leBAduOR`x2#%HA{EDWj~+keOp;WoKlMkR5(6 z@6Y$wt$+GQe|Wy0<9b}zbw5(hb;!;pEOp?{LU;T#vfHTw!k~y%BZtbPoyzOh8DfTI zRt?Sb*DD?Ui&_8Y@%<7p8s+ zJ@h?cO4BHL+@?;g@aRnveL75SFvxBboS4I*&Q>^LCB%$Cv+6v&%lXdP@s&404ZytS zjwFWb6wbO{?99vZPE%sGX^%5^O@*$|Lq;#;o6^ zNhkLZ1@E=CpL9F*ej#0=r9?E4uQ*{Y?p(2dMWUqYmQQeu;J!U(nB(F$G#(qlLlq8H z{H5gYz3}fIys12$*`nn^@-n+SC=UxJ`_ndM^8ESCzDz(E;_*l#DeGS2LAVRMjX-)~ zl8A_idNOzCh&RvyiUXp@nFBUu*!!a!-@mI$Gt(rg-nbGtIUT96S7o6eO|``t zDCu=VbijGjLTGp-_giWy&J@7A%{#0xUH7eWQ9hKIoZ;`aGEh5vCucv}wi!TJEdXVr z7Yo-{^?(5ks3m4=_)+}w#>#(Fbtdk2`sXgtVs#TYxd}qkH>NE2!~#JuzE;5P&{*NS z1!S;n1CMRPDVP{?aG_TXKBwP?_GQVg&LrPfZoSx22#WianR7l?;N|J%=j8jkA?b(k ziTi+7u9we-P@7|11UHBvfjtCOt}zmcvz;-%TV={LTkRa{=pgvOh`LJA%$bO%h_?fN zdE$@!MYpxjRV>6DYXNE4dN=Q7<%_Nk-^TLrutid@3^#G)Y3OOFBi<>hcVY-9G)ZaX zmhABCo{+IC2-;niAUD$-&&upp?|7;2~ey-t{T(2}2w^K`|EIxzMlhysQ?stJki za~tc@WMRzq%V@b+5xbe$qarF8oKUgkR6uv~H&kKLQ$-I&1G^*0e^0?fxAS8q5|h-m>c@pEI&ylEyu-@a zgprR)0@y8Vq=!in9m?I@?B!@p;>Os9l3JS}D5I>DHd#kiowTi)(iDA6Q9fH2R;nMD zr;0+MZqW)9>+RsybiV&-6|o!6lMlTz6sTUf+F7^)e1s$A<>fc7eSkyM1w?e2u~996 zSC9`d-rsd@-v4(v^c6^Q4y~IF-h`T2PJ>x)ip~HI4d-*9WAtW24Cg$|3w&>5tk^Zq z`ehmIz22-Ccs^n5rTMFiVaRuOy~6nIj(CgvCY1zR{wJGnue#!>)rmsn$slQ7b@g>3 zHZPx7?ifD(gdf@)h9h1z88l@ZlVl}CQd`nk4&X*NxLoX357|$cUalR=bXfWuCS(xV z@PmWJ1)vDDFhh|Kk`0P?{GL>cNRVm+`e`LZB?ue@zqh7=Dx#*@-{b|M(IJ?~$UtAO zLZ_L!yLU4OK5~nDnGA~eXv6Otv{{3KA}n*`#b27QB*YKXHl&Up)izeZ+GAfK{5V>< z;BW;az1YyBAoH(AFW!%b1|**!KhQ<->sG_N_P@_J%nK$A;M&xqGKp^43BntnXg{OD ziv)%T@Gies-yhV|Qcjix;E+B)5r3U1cyVGqdMC$>yUc~tCN8zjcz>78)Tns6dSwpZ zRDEvLQ3cYra#c_T><5FQHYtV}5mC{}i3uqY-PmrWTC6gLwaDOW5s)6F)itQ9zaVhA z8?O+}jp7DMXaE^hhep69P5|E7hRd?RaO-;{>sZ{^Mfcc>%oqZH-c|3U04leLAPCox zw(eI~-O@0EvjLKSKRliaqJ@6AOFRWEQc9}P?L8aJs^uSUHW#7w4|za{GGOS$Z%EDb z;5N#C4S)#&ZhB=(l&CUt9sp6@6c67-Ic~5Ef*JI)LIXF6N|`OgA;6vA6mSk;tN=W7 z=dV@B`LmJboKIdq!zx|r+ZF?pVp~prjt9D&?DlUV_N4X5FVKw}W1Q|f`1c*;L8Wy3 zaE2$D;;zxt$)3xnq>TIC5=Ji;&@~TCX2|G*5?vrtaTcqC8kh+2ZUdADjio?h3>9$U zmd4K^F|6S&TgT^6>@~}KXU#^2**+!#YPymjWW-8JdJnth)RAKmE3@KpQ%_^8Eo)Ac z@q#e$cnqF}Hg|qeS{YmpqZ+S=QhVQOl4%jGLpWDx@Z}CB4$EOMd#Hk{wcVT7yNB#G z2@gVazZoBWmRFFT0V6UlOu%S$`rLk(u^gA}M1W~LO;3*~&6EMHTs{ZI>DbqLBt^d% zd9f8hMD6MxANe};3!A`0Z_zN$MfV-j}6PH%Ff)prjM08yhL+|=9vLCj>b0Y$BV`2=4LO44})G9v@y2#ckcN&euy zy@j7<8R0WSK?<_e;$nV%yH8!OD_&ItacV#-Sz0PeRle0YHTbb&%ethg$*IaN`Wecr z7?}hu@AVT|n4iwr*-p{{Jwq>&!jdfjF_I!kLJXu2U^yYaD06TB8Jc1blXaGz9{AQH^y$L|*%nM+k-9QzY za{5BAYX6SiVLJciJd4==Hu*&)c9rTx+NH+6xkE+l)u7qBloqV2LI)8K=aLJi`x(r~ z7zD|jS{-3eS-Kjrv(|H59WWfhU)Lv8kYjZtQ)8{rd5YuKM`eFrnHoF>P$@00f9cFw zg7?3W;bmfy6mW0N@>JBr$T0SutoQMHRYgWx4eth}>qPcKY~T$m+tn(dU!l)%cEp3~ zu2r8fx2YK3#<>|oVVF%GS2_6Rw0=}!n!0eeU6jI}i77EF8_Y&q0YBdW<={IZ4h`{e z9HKi(+0uzD^O>xrWWl3jOWT0|BFNt?GPq_YV~RuQtp$$QLB<_2xKECF=BfItglQyIpKWW^OcBlJyU>>j6HW z*D!MYvY-O^ueggx?8Za{@#U5v5)3h1altV?1=-nWc0$<49?{&zLYV{4^q=9etvyHe zMf_|{SLfv+&TI0WfZ+v2xDDEq+4FRC=412k^N9E8y&fz*arL8hc2z(XzRUn-*|1q5 z_+u)=jOgK9dDuSQS8N5IS!>HfaY44*R6v(ngdWElFWiuf0f7cfHCFWY>v=z9RTnjkd+&_RY;?QuilcOVE8<1SQ1-ilD0fCx;<$f(2 zjl<-@H8kjSI9!Gh1e9~CEdZ~wj};XZm7TzuHxro=`F_GE zBb7TupgX#z3c1)DFG_;Q*>?v&Wm1a9jpPYGU~Na@VBathl}@K*cau8`cI{0#uXJh^ z9Ak4&?~_N!`%v2Ma9xiN+P7&09cCtw>_l4Zg*vKzT|TNqeFE5g3$keWV6mco_uKGL z!xSazs$CfxF6Lg=ABGqT3E;sV+Evaj*OBItAZS~Pp(f&a|R zyeCju7#I;tf2r!+xKRucIM@EZ=x6!Y&eFWD#K&M7eX|{HVhBwCI{M1~%~TFAr-NjZ z@F-x2m%)W&X_3(Q+X4!2@uB6aX=~P7r}x>FC|(M5tMiz^sHqj&m=Dw#^YP$$rL^2C z-rGNcWcIr>Fz8Vnf9T4g1?>dE#&Ah()QD~aNVDO~ zC#9~HooyKIuwS)HY=oM|yd9E2kHciGW4~NgP&Gz%a7vmIf2`FfD-ukH#j+#n)+fw*(yba z)ga99{H@@<{F=tb37joAF_f)y#BR1`z}q?lf5^rWH4aF8I>}fvkd8?gT{<`6F;NDF zwG3OD1}DMSt!&1(Bq2?=(dso>yGEdA>Ub2wg7ep~(3c(_9+qfDC3V7hMpnY0>ZwvqWm@2)N=YLO^ z5T`jA6FfG|2n3pr&AlIZ%~Gm ze!mEQtKZ+Pck~6yAigfqRDJtU1rd&Unts#!EMMwq!C9 zn_iRzq6Vck7_2EODka~zr>5(6YDDk!dXgJFd{Co~*H=&HsU3@VetvjSq(=Lfk)NM1 zTs!kw%`#kQOg3T+3Pu2L26o*+@i;uQWcw0crk{akXf$V$j}XHf=;fdy7}1$@8%AhD z^k?PA!FH)3h#!T>>uuFOEG80uyg4o6RUQ#ZO<;Zhz}^RbA75AWK=o14R$K@zUTB?T zM)6r=?YIJ69IC6HU_`B_kW=wZp2R}>fSi@Bg9EVi3h4~MY@KmB790eCtQUzX8~PrB z@x}b_%K89%lamLaPor0|mB%-+0)W!n`lsX8i_6Vtx$htN1Fg$vH=uim#Vo+0e$z&C zH)kfa6=(r?T0iSC*#fK(dFdj~e>0bXm${Ubz5?ro*3kWg>y0+$it|{nK=WS@vOOh^ z9zH8(fIYZH`bm`$4wl1HVi(B|i8H6xRpoLV71{Z+q|L=GKxBhd{>iGNqCFnjdqeW% zWc0SRwMAPhs>;jR?mim%YQ|Fp2y6^;(FkZs=lX9g422v*uyX4@pwH^sl71-LSdu-Y zMZ4{>hp}SA^pI|YJdtum7;HCPgrT0&c=i4Hm~oHn^awB;47Q%1b@%m1sZ!W0Viam; zuHLy(fE1^qI@0q6pY@1~wvvETgn!eEI0{qt@K+}|$aL%}jwb{I|3UJ6OaX(TOH2Kg zR@4`QVf&My2t=R_Ad7 z4B-L}_5njfxUmL(MUG7s$vPQleszKlnXe~Bo{qYk5ESFM;PK)`D~V3(D=XV)emPCE zxHWwtFB8Gs`lslVIj`8w$b!(fgc=i{2w+F+wn$qZ;ls06zP=8?0$E1Rh@g`7^`Gfa zPFScGCH+XY5g{SDpbL`cwl8Uq^0_&Kh|gv&9P8&qw!Z=Y&ECf-h!C0Bc52nF@E8S~>hNLFs0s;;W_!NMVvr$nvVVStB%AaPhM{Fd16lRMua(2e0K+r6o#`U(d*VQL~JXMN?9CqbBY zUc$%hB`f^s4GCfN=`ENfjQ&pC{MA*9_>(t)!}f4T4y0*|V4GNu z55jqR{uFv6G4AofNV9u1d3?D?b9<&<_T8r5vx0rGPJ?j|OaKo*p^XB_8cF39u2oBj zg!L71nNI@IOw|!MC@+CD2>*kQei@dGT8;ZkNRETRaV7;ibLN*R60r`vfgZxklazt> zW5Qk=V*Fm0&r?k2)sG>d;3;*$KnGL}OrJ^VN04?W!fuysyh7V9|3t7>wT?lcp-K1G z*x-^k;zx-ofHZ9SchC$^Ph>MswG#kXBIq$?P79BU7OBQ;(QQMkX`|CA48IEbdiH5qd4t= z1+utRwihDySZ<~5fx`pfFPqM!$AI5sxiKtU{!dIRPpO``Yz@$)iayTH=6@LJ2j76B_UgP3L&(WS+u+3n3AS@y)1ao^S~dGOc@R{* zC833;2Pv^C!ZjG_zrY#qX~Yv~bKfhZR)AKI*N#hH9EbHf=6E?OhM`mw`+$y0lt*or ztvtv;%R`%}6B(tVJ(Qo6-;l>3uKfm4q!+7vRn(%np9V63wEM@*Ei9sh-v-Hujd0=;L^gswC>=o;2EPhE&d*`$Qjvo?|F6k2#hcze>%N@c86nyF5 zVQUMPZ+@2sf{i-U6^CrX6cFNg-Dqu9`E7?ulK&o`+#`N6*FaI%{>@Am=(rggx||&{ z0SJBs&7S{l_}LAO>l?nY^RKt8{CBb{16P1O{k_IzFUpLUxPiLTG*U%3sV+bLrG9T1 ztsC@_5|>36N=eJLJxO9Ydw?F+h1CHerM8CK^~M|AmCz~(Ag?DkY9mtln0*?el1&N< z%6Ex|d}OsLwuj#q3rde*Wpxc|wIahrqz!DZe_Eyx(TCZPb(XvP4|YMCbNbo{`Ms&A zFxU(B0$1T&VJt0y(1{21N&i- z@^oRx9d8{ga(oOJj3?HM_Vhy~JTFGt=1jWaow93rfVqG6Lb-*4rmDrkjWv4uXPVLG(q` zn{z@(1>^{k7z=Rlw;#8*62*}E)IXtjP8s2N{3s6k*vHdJ0t+L{#xED9KIwiShqK+r zzi3RSheFiRn$`k>G}fJTKjS6LcoN2Y#>e!=K}a%CSev*l%$520|D@{@B+ykjXPs)| zIr>yI8-ObSAkgvDJ-r(V;^r_%m+!_bxNMx402)<~?A}X6m${;bV)l6$1>6j5q3@=T z5Ks+TmP5otaqW@mFan084mp~h$ZuT5L4-kUgp{6H9)HZh<|3JnPR!03hA%!4QY+F} z=C^gKZ+B_h5MtEdwZ18$w*cGO7f*PY3@YhGVNOsD)5U!zYX~wy-DIVxqzeH#m3%mM zc!o#g$6_7e;DTGAVh}}GhH-YnKZYMFb{gwUbb?_B7moU*NbX^~Mz`Gt$`6}7VL>Xp z-2ByPk4A97S7MXDl%w>O7j&O8gFolhTO&C~AX}`KHIQHNfFI4>)s(2p4f?`Z{ayTS z5PPB;Oq(d^`GMQV?oRACuL1h#vJoZpQ~FdMj<+gTK6Db4CzAfpc{khd=&(Ix3Em8 z0tSghu(Bt>%pr(wV9V}qthi{gSOt8P-1jV4#^E-rP_bgIr<{rVk^6%?UJJhc<;t>N zYUu-hY{=%qHQst6JZUzRsz_B*f-w@&weLNV*@YE~Tu*`nYWJc&}V1Ym$pl_j2c%U?F`ZJ4*zo`WQ=|7Fj45-s78O5p^ z>rw#O?%XaZ;~YW_3SmoBd^UX1d=bHkLI-K(GMWQD5+5Rye#Ak>u-jmt7o?Z`dSjy!$S}~mGKyeb2%~~JVw{^lle*VXPwHzYg3DmUq8J3V zvupV8YpulIw2a=MQ7V1swk9SgZ))&v@H)VRxn$ceLsE8s_r>}Cp8x7HbHFiG;O=O3 z_{zJ$haX-4`_c(^0_g&`TLM3${S+ganm#aztO$*ju{7v#fa8*?%=GcKZS+E^8)$bR zQ9*I{I?Y&h1XM?VXaw&VDpG@3J8N2PhgQ2+KNTorrobhM+^)KA^s)9RJVN+mt_Wqd}$pZU9TIp|ht z_-jV>Xq7UoCAfO0ywCl%q8zfApE(9vnv{UG7cwC&qxgiM4<$gmP~jY@>OEY7%k)wF z%Wa?)k3Bo6r4Y*IF)PeaCm5vdrmfAtoHm?vUJ_FlSB5Byr+&HDw4=YX>VRr-yt`=wdJDK9z8$=y9o`PU z{lO5bpu_$c0k4usBdhB&ddP@RMrZfT%J-dd$YF(E;&hz8zCP+`J^mtNN*&id`ed?( zkq4_1>0+ipm@J3G`6HFd<;cg=AZ=}t)cGnZa<9NPWW6~H`*Et6;D~C$(5P6}*9a=M zj?d%04~oX+08_`L$KjcjmkH}GZ+My>4XNJ4wb?wZp_M*Vc}gHY5}%Hh{(USLtOPI~ zv@4QWCa1k@@4>rIOT;v7O3B|Ktm; zKMJEa`2-!ztQoCD=t+*jqf7f0@(=%(69rpj2BrEL%%|(8n@F!$<+W8|B>22dv!6^Z z=xvYpbauEE85a6M47=qiLRl7BMux~(&d2*#Xa9J;C*Uj3Ls5Mn z=UXZ$P&}BvY^7O+lV%~%0w1JDq+Q=hEB&YS6EABt7uUeV>hWNNG|6O%etoqe`X8an zV(;)n7X^$Cnb<&hFS3YQ!3xezpT611V?*9lf|dy3vbg=f+P_6Wi#ohn9Rk81_oUAjxTaMrN7 zKXh3ic*v=obJao;E=>=~=F3m8&?dr`NbF(VNPhSK=^4|e;7d$g2r}us@ zaT`LtB#|nUv$6PP=0qC@>fRmhdrvK+qB9gH9my^Pbr_Ez^ z{WeIP)>GUVG_D}H%nrt0c+}$iZ1S17mU??h8Mtp)!E8%VSs{$grG$vK%2Zg6L+chB zzcP^$Tp@OGy`pY=9&#WB3^QPVKHak-0ZF~kRdv1fM0H=`9!!Z)mrY@7j_$8rZ2A%T z{@B`!fAUYq^jZk*5k$5wZoK`Tiq~fgXqV<2+-P+3;$UI|=fctPZXCZkdM;b-RqmtnFKJb9Lp^e$I1DQIV zV`J3mv=41IOAF*aV#U`M{T**kn${lxeplu%fKgomn7n>K`>Xpn>RI+Q&xd35p7*|P zz`Eu;e2|<+yVt*;*XNt5{wIRWhjHIuXK6TWJ6+DPT(-aXvX7VQ7XA4jjo(nu{P~Zb zz*FbzKi|Hdcc7X788*#c?WRgybIJ%dM-ykKyr5t={aYg~>bHGbMHAY(JRW=c%lY}y zk!G{qa;swC$yz=6+rf=8-PoD+xK4%l91y}Fh)L#Cp;DzzSMaEIB&-Qe6x5bxhn^p} z1#*P}x}(x}pzwQeJY-qRXtL?{4vp(8NxRH&kghxMf>{an>E`y=42`%sCx-9Mks zGI~TUx-D-@f?Ee!Q;La9iwCFW7jHnp^_qBa1nj-}qzCnvenUO+?h(71>gq(SP|iEWtn}YnGnD@O;>>SjJp%ej$C`hH@$My#y)u7?F@vP zfi^dTQ4yWUl{}1(+XRlKKAav8V?XG)D-cwMq2(6Oq(`vFKiriqTAA7T$;92|{tHg| z-yBry>1gw+a3xSJn1cppo`98@9H^x3(R4+e4VDC4HSu|sSYFP||9R2s^6esE&0>zAR5>rqmk*Hl8#YDTzxz4x^o}kW_gfVt zodt3UP6V}3ZC-CQG9qLY3kyQ#8~Isu#fgW-r4?YYDt+Q!hEb&%c9gGO1InpfQph@zu9)emrKVlmIP4vYo#qIE6(Kmb$27+Ow zz}LG6ej^lSR6Q3!9Ku12_9H0uUk4UDg z8K_q$sDU{K)1dOCAW|31&1zxwI8?5ea0UHzjNP}HWiF~sv%+bFY1i1^ET}gauTT0e z%R7Pfi@}t;zfXp*zdiUFNxwp{w*B=qq2!g2wPn)=T^B@B?$EI&k$L8`@IbCx5gk0} z_w13S|1WV?tJQsP9L_8VgGUN-OC^!D{cl5YyJ=$sj4_<8zaKh_1Lk<3WNTBjbw37? zGtDr0aQ5KEVU#ExDEjNp_4&_>h10C+QAMXo&2PdwvxiA6txFkA&KI31+3RoDyGzO+ zPo1yae|6kxUNvF)dYrwmai#7x5O|vY^@}Ep``` z0lC0z*Z-vv@l%#Se?;MLbf?hCtyks!yQ-mY82%#*EDV?UR6SjHCw2O{gHm zy`*PZdG49;NbvQqQZIYJLryHG-Wld3j+YjU;SKJ=PS-x%$^UsuH`MVQB-r2ZcUjrg zx=0m?KQI3dG+b-4+nBwxlO=fciOGQ0=JmR$f23`y7eN@|p>`+$8^R48E%8UOOQ6N^O&6Wib;e>%?cZu5-3DXF%5SKo!C zyIt+WbQi%2t**Cdh$ZuQWb|p z1OgiQ(srUOpNC5Q{_@l-Wz|RGeB4bss=L!R^Uy_iYl-ZNk2{94E)#}~_y|5lw&FTm z5Wg8DI6Zy}gnX)}H~uyKqTU^4{kdyaXUN#_duFm2AqNL?&qx6&O6mi&6jLm>5(}on z&4`y|h}_I*gv<(6kouj}-H98))s*@kOF`WCkb2rIa7B(2HcTAA)38|z=Y$s%Kek!F z_X5W#=573i*SnjIke>?X%w?NTSD>i}JNyy9ufwA`m1d}o5U>&1sfy&l6qKyLo3tGX6GI+@Rbq5#^_HQrW$ukYljWL$9x0741 zpfO|Y!IndrDh>}%(X4p>n^8_4bI?i()reaXBqD|Mk)Sh4tcpsMATvPz9W7F}y=+0S zvuU%enA_-%00y6I<%+XI%7uVs+a8Z;^dD9?nfgjXwuE}x@g#fa|5L_OHz$mp{U@q(j;1GDJsll)Vy{nQ zZ=74YhU%>sGn0$4t$Q}0p!0Ou^SL!?p}F8u=a`zS7|pY{M|>x+43^{VJAu`KIa7^i zg5gbz_k`BfherzT&deMFM-f{`PUwMV?%5W>3$NqEoa^Sy>(wxNsnw-g&A>mg!jvfs zFH3Z3q)H#p4^H!- zr?o4et6N6v9J3YT8SyL2_voK?H#j!(@{n=%yosdOLv^vSst()T+KiY?N!3vRgBj>h zAiq3f>C%xy{WL?oIXuL^+txg<%kGm{F8$v`-byG)kal|;5;wadk2)xfO|h_0Y2IrK zKDzYPVN!%JVxUN}5QH}EK9U2*uMK`rq@88||Gfaz_c%~Wa)MN~O7~LAc(m8GC6GbA zEiIoJW8gi%heN#B^S@v^Pvh>~=Hbf21L0x0L7tAg&P7-wtC7aqD%y`227`sc-BU}- z0?fmwRQMow2Z9zxrgKEiZ^@{7`Cwnm$e^I*eU(ODti39u1$cC$Q$1Z*TfEA7&#F;p zQn!cvBH}O(MzXvA1SumLzV58~7DM(p2+NQ*w-fmet9{P`1*{+*9!x&7mk}?=wg6ri z1g1phJRm9)xaOGK5M^84S3r^DaV#DWC#HUQ<+A%%dyk77qcT}59?C$));4%N@k)dW zCQP7hp~Qzkw(LZq*wO?NUR%^#*6<>>Y zB4v1i?w#ZE5H~p*#q9<`Tw!r=I#dzd8Qlpn(DO78*IZRkGwd{5Oei$~>)|4k;zDA7 zrjdfMY=FB65uL_B-E3<}OD* zh!)=!g49Ey9(CQFnyc>!kW5OuJb>TEUz>eR(3nocJw^V8=$0W8b+O8;%f8Jnim{6_ za?iDL3lA6ndrS*TJKl}15A@xl^|?**DVKuE0B3$w{^u69+S$hBndDvL1}h46+=oPz z0I#@1G6kblRb6X6b9tx1nM#Ib+MR04%QCiAXow{Dl|hu{y6t>Bzd3Kmbr_kjURm@!N^}rNCf;leemM!`$_Yf zbiv3vKxV(7yd0*??yxxE!+v(HJh2Jqc@+EctVH(g4a-@?i+|O|p0&(ZvXg#)yaIP@ znXfutG#)U?o|W5XYY$%2H+1jjTzeX8Dl(VH+Ic@Dm)IE>+F4yauA&SD{emqRe3(D; zSol-gBXjlyuzP3<37)O09S5vd${bhS8hj9V-C!Q5A)0;u0%0XlVNWxgllsv>fu}R1 zEtp@2v#6g81anMM!0KRd7&aY-t~y7+c7#qPZ?+@FgV9G+RbUhPyH_8N*4)iBuAb7^ zeDL91IG6XE{zIVf`mPxUnEXw8Bx93!5ZYYDKYnj7Yg2HP>yyKF=lV(!ojs8NhWR zKBB!1?>n1V(#UX|WVOF3Bt4z+=yO)E1bw$Qio2*@khbf|V7MvV=Hn2;P}?Lr`h7YZ z5r|9aUk-Y=O;1&xu;a>-8aCB2U})tT{c=)^<`GFx${K-BfA_0N#7Q56tgC6f5?)i5 zTHm&K8QrsI`XZWHS|f2k+#Q{zP02qU(APle?i`uU%LU%Hn{)pDL_zT372s4cerB&P z_0t*zu5&qQhP^uBnsUqTC$8ZcHz}BVz9}|;Q5v3m@-S8{k((b3*8Q{gaX|~GP7Z^} z>7m9*z_yacCX+t;rOYv#jUuDKZZrYGV@GcYxN3#@NYKwu{^wYKQ9&7a`t6R1`^XIt z=!Up<1OEZ&ECG!T;-H}S2eJc9Ex+d}KyvjmM`}ZXv$w8KF2tGC=8jt@r7xFKudjA~ zt4qLaml$S%pX0SKb|D`7<(C|TsYd7^bRYVcIfnr6B&)i?V|)In`gkrt@anhoJRT@E zaPWM={}=lM-^uXH8({~YEcJBUN@^tksj)z3itYy)U+05`LD@3bTcJpz z@7)p0CZp!gcv^d-U+$R?*ADpEmbl*oFo&IQgAoZAME;V(KJ3CO+>WE3CL|a}1u&uz zgBL{$bj%bxPQ%~e^KFQ$Py?fZ> z{wc7GPwixWxme{|wB8tZI*uy)e(BBM;y)4Y4+Wb^q~mNQRWQUjU|Jj>fr(QTHf7SB zv`|N`*BLFq-bxQ(?s&e7?Krc1xZ0dGz09=pL-GZ2QBq>@*pFNz_ZYsI_g#-j@mb;F z{1wkQNhrrj2ItXNuU}l3Ys&mhyTwWj2CG=r%qISJ)7#`C zWrg7y+9`0{Wgcyh3i830@7t5w{|xP^K;}62G$PmgxH{J#c(DU;vfP%{3v}? z%Ebyz|6kZ^pu70*R^aa1^^n)UMb16@*3P3JhOq|{MZUl)lav^~xLWElJ=G1YB?VO- zJiP7?{IXf;+;}+OBzs-IcQMWq&~2f$J+bibgzt1g=Jn<^^m?JGQ_v;GZH?s$mwDZK zExYh%+u4Z8R5x@G%p_40D8WR2@kh8qZ^Szzb7$&I^R)Qn>yZd-CQw7#aAsP}b7}bX z8pQ>gVN-~%0lQ`6U~4S^`}Vq8xP9)efQ_;4Dx3sV+{aLqse@Sj{ZTu24ixnMO!0Fp7^&X!4w2`4P(} zzo0sAX*Z@P^+ld8Jv-gOMs0|z(?AsuWG(_y6x!ZsuwzXiZEwEnOM*a#{o&d~g@6g7 zvZB0rC74y9wvD%lK#-rK<2tp4kG#47J7M~*LPD4FiS2y|TL``YX(VJxX`c*S%KDsB zrXqixCQG~ZL4JsOzL;hnm$x=G5fL>t4NbhEoC*~D7;43jZoGh?d@E~PS(-N8L&JJ5 zBI1OOWX?wBHe^r^&pdLg7Y->H65nQ9uBqQM3ezJeO}jNs5}&_T>V8b^hZHg>OAi~4 zC3zR7#48Y+QP4-eg6ZA1Sc+OPRYH-IOJonOPuI%yGg?$IP#xToAV$<#91#iZhZw3a zanCH^L4wIGn&Zp5-B%PEZ>JaAHAFVvw^86H4Th4^gGh`BV*fdwWldGvmrt|}yHpE< zFrUyokGMm+#Vzh-<0QFV^^KO#wAp2`<8V;Y8SvKys!~?&00px9@rg1gm&TJ)n>5EA{;_9c0QJW^ zswd1?NOZyR1H)`C-mGi;?-Qovz!QQsDMFA_(}{^&7v~E^rFXLER`Q-t`<1izPG9Bh zT9`urpAKfpO=P<*S|4$XQLNe59eLSx|FI>4qs1L8egNVQRa$2sU+$F zw9(jB?XM5RCALELg^fA^t+54{K zY+J`Zb51GN?H61o=JyrgaJfkGsrZ+2a;SmNrt@3YY=^INJbag0_RfAO4dhF5zlD6j zqBZmshWq@K4Qm}oY82x;@%RbgC18dpkZ~?3&p0y~nAqM197IS=t0(rvW+;2cUO1im z>20_`B8NGZ@VLqwP#$-8hsU@IIo?qN>1wYOp2bRXWNx~dt(b^Gx*?RiQyK7P?s#74 z-BA~iLo}6!Mvian!);nVjxtYrg9VY1?>LCIW-_2`jW|7`d=M!N?|hophFGyGABFuI z6=dK*V;PEvA_qNBz)dGp=8MAH{)^72?dXeo;YGM9~|$^w1LIDaG$RKfAUg<~(%gV4MDJ zFJZ_hMpLGS5SE&&ZqnwH2z1^w6U&Phd+R?kz$-zJjLphW1M@QDwr+f%WA|XXyk(v+ zzmUPD$p@`P?zX3*b2XzHk|qFKMK|C_@`Gp0KI`v$7}2<-Ts_o~*mAT95Xs+P>T-VF zZ+r!m2S|;-(palN&;M;Mr<7Sl=#hX{0oRSnwS|#3792TA@(`d)>9gtN+Up!^!<651yh8${@-@9t5&1M+qdKggCG8m$dJm=g ztDQvmtY7n;%q#0CB9ZwL1WTSK$R$^DUMH*?!y5}r?b*XjEMY{M1RG)QZ`ZBe?<+~Y zq~_*}cyd})d`1p+HiJ2+4oCeh*DEBAtQPbcMZ0` z=?fw^Nq!@3I9(~!REcosuwHF*LL~|fM5XFT2lDM;5&VsZno{GA6Idn;x)B|cByV+p zDb1C!3FiS%ZnQ$@n?Gz#%VFHOJC*qPq72L<0unG=QvmZQfaYruLTPKS3$$%LLxJY% z$8*q8jtD)q?FxmZ09B2Z!pLu(y}zI2@FA^u@DuV2%bZbBsnqwafB&U#ks+WVxBWfY zep@X(eD>dK^Hu))g9vej$Kjwz`Ed~0h+Z_W0zb@ku*|+*aGv>Yl(;g2g&wB(2gAw` zLg+~W#R-{*BNe}mW|Z?NmMFXle5e5C%y$2=+qBne)t7LCM6;*#1?oRu0r;SstK&P# zU9noumg}pFlX>4g0CyUysHh0!+hPZ0F1Fj-OHx>duKfnheVJY7rO812UoP3MFN;|I zEdflIF3Q$_3ze^(sLOxmem$w1qmVjk98a8=U5h!9wm4C7m_2Mb3A82!jhPat zzqJ8-w4}oX81($8C48^Z$>YD3Ia}D;Ct_Us>Y)Cs6}YPZs@iYweYY9fdje$;8*bTX(*gBg?}uIT~(oihq`&orh%AkRp=>qaX<7(&<-}N z&h+V=Ro-v4n%6R)ck7LmsNjYu5MR1YOW*P~55a2KJmj11ooj0dZYSZBZvzGAYjolc zqo-%O=YhZN-JiG0`ATo5m8DGAs@(+9=-R9 zI7~h4ouPsjN`tPTtA{lsC||#bl^2S~e=64*Fa18>|AU=oxWFM!FTPa&1I=r>`2C3D z&jg-?Vuq`RjntFun&ABDqN zA<&JUS%a7F*a`EOK9kT0h8cZ-$>96d@>LaH;Y%axeoOfo?duombaO9038Z+3*zg$r zz(bBh#z0PHz=W20O|5->NAyw*dov_lF^ZR1jMm#*pY#UnG4$haW{(g zq0iBW7ne!73ax+dI=}9T)dztf&8M9Uye+>;N|^k6`;TW2t-~2Trr*@_p7kLlyLLlI zUId=fT(4!SafWd=_kEc6`UM!gPn9P?Al#hiVX#MYbAch>#U7fp}3h^?6L2d!t5(2qumw!Etk zUyU=wNLJ7vHo!aj=bQKQ7v@iao6KpO$=uD7T-#rqd0jHcM9+vV8PsDaQv-GP%)Bnn z5r+z>us==UKtbl7Pg*q)o3CY|gH*QD#4{XJpS+>R#KUKtH^IJ^yf5^a#B0KfeLn4E) z5gD}O+9UncXuzU1&X`XGQ)JAMadi3e2ALOz`Sj)c#dA~^I#Ou@lJ+S|c+2|A)ji8L~?(PVHIN$8-ErqV_j1m7vlrtWzI&eU z?~^N^4K2)Uk5bQaE&Dm5v!e(1e41~9BZG0+C=-=YQZI1?4yND4LIjt9iJ6leid-e@ zrhy;1j<0|X)BAi_SkuON?~)m#P=FvJIf(j}`&ZuF%LMIYk)g5!1&007Y)-K1baQFN zPais-DQTCPVEF#II7S<`1rAz=kJE6B!!T1^B&1bn=_op}q;Us%v*^V|UPwh-gO3BV zOPL8}O_RwD)9oxNLyQNeF+YF4_eRbnNxo_3;y)sa#(U-TU&GpNVK^zLar1G{z548X z*N*LUWa{BuF|SZ??u9+3(kT8s`f82hBa5AvK!(-bX7H ze?*rA0}-Fh3?OZ<7>|szupQLe9^^kO>U2$sf_RPRj;jZb2E<`A}Eb7>yTV2M2;WCvBhT~v$L?S zOH4(NjH4g=>0*hh$wz_%(5xwg<=RjHh;3`6x25FmV>Jn}B12ixNg#3ezFdsO->ZoK zG5~mGcvyWvJEQOIFD5cEj%U#U=b~++o|T!b&}?GDG~izXK1~KX)^VJ0bda3;`Obuj zV^mcb$pQ3jbv!dSQp={2St;lXMwQe{0e>bZLV#7IVQw{Rt~Tzl15;~lF#SMoB_+wC zFvcZ=C0NV5MC;FnM}0#zGLtYiF3xMorNG7lHwE`gZS5dxBx^KygEi^za^4-hIr(+} z$f)&(L5IaONBJ^ckq-!kZp1-&hT1+M4oeUGf!9PSTi6za!exmXKwQNe<)78JrCAhJ zx>a041L*50eY>0mJeCA;)s;hpKgY<)gE1MUxd9~H?%0x;;>Oj?a%vgHjA%{dJj7}a ztLg&3{E#+IYL+*t$7x(39yuaXkqrO0mhvx6g}DTERh(k=ZY(2lHN?~3ng^ZqWA$Fv zxnhL;Be?$E%&;sS9e)mRMV{Bvzv?TbpSHssuo?W6 zvgw_p-ylT7Pu+DADFVw8paOho_cUsuoUnGjIGF3Q<2$~6Tx$9CaG*w2?=u3c2|j6> zo9tWIn^S*VfB`_8{w%+vkHLYQLUL)W@XN+iKC6S@6g={TDYhlzE*4zB&c9%)tSptL zQK41S$5mq!(MjX+#h(s`BssEq^&aG^^u+?QMup}n$z=zDMmVoUxuN!w^1M^V7=@5Y zWk_Gl|fs1$Uyr^21#`m&X)-6 zKFiWK4XcJ1@+Y;B7*l|wD7$S8o?zD;U^ zId@0w0T;jQII=742}g)riC@QQ{Rb6?;sSp6N!IWPWz0>|r+>y`p*TX@l=36S6ffU-HC}d< z+-PEAf+I`nrvGLe)tPyZZ+9d(z`)6ZEQGo(`Zb^=KPgXnQ&(P0Of13-(RZCmxFqFV zZ+aij4qKOxHe%kwtrEnLMb51Py|-gJmt=>?dwJQZz8kqHwmVFhzw`TBW|!}4>l8ig ze(Be36)$9qER77>E&3cU@pSXAZIbORb1?C62K>SR0H~C>zs;GOKaF;=J|>{;RnQLb zIwo85`*~@#9Xu8^#M0|C*?cCF5wN-68yxoY2#A|)Z82Q9tSN6AY&X*)5VWCppR0|I zXr~ePx!q>-BidE_bhTgRrhY;jzxDVL~Isc~iWq%ZL#MdYLz&)loyLg`f@-aH;F{gTBC@e5J! zlxzjffp~(@cc``4lB05@+^Ap(kxp{?8>M2t5!EQRWRWpjo(N!_TJ?X_M%0!F=uc!T z71&)XR0d;&=u-TEngvIMtjRV6TbT_DtTmq)n!)78K-lxr+tSp@r}&VUzF?s~JL}rNy>8)I23s@;Z8{*h8snjA{&Q zJPp6c>9su_q>`Rdw~(I%VUWy6LX^moH=7ThAbMvDsC_m4HGA@>>P%b0NvcQ z>PP5b6(dh@Fp28CEQ;@sz>BKfF#CoeNmXC-bcyg3Yren=eMWIzH7Tk~d4FdZ1Wnt? zB9XcVrHLnmRW|xHDZZvICZ|+Ye3qv@XRTp1wj8|y{c;q^WwllagE0O1(S03r*)uRz zlm^5+UiRh_EB1eYsprU)l!77{DK|cRWEQB!r#_ktPd?HIojc^qC z1&07$YEXT^$7^Twy$Cm`)cjK*T`?_$JjAZnf}fGk$@%U-+5VXxzDK^O*!#M6H`msR zTnX16kG!#7`K)gY#Sn;|^i>J}MP`rn{Cd+Fu)|ZZ7;rxJPhygyF3hHsTyYu8>)CX3 zz}oG+8b;W*SQEd}9bA5$dwMmoGWXis`kdf)1#$gWCzn?6{L#GIiUu_u-raj2cwn() zeSMfOO!7lQV&hK8>i+62_1zi4^BZ}Ub&vR-8@q}_v6@B(-GxFfkDUv$E(?F2_VaJE zPU+jTVWK6j|J`1a5ZGL=bhXZHaC&+5K8=^r-V|tvxgVcAdbgs4&=8E}n$>B`?LMqh z@J{u1lZUb0{&Z5pZnr%brVNq_+jD=c*(} zEV?DD0!ja~h6310eNz-k?N4kPV^IJQC2ZSaRi-*A+1NqYI07CE@VV;powik#k z!sFm7f+{BN0(tcGl!!7?x7DUZF|ZY74`3W>L~;1WnyL$LQ=&tq=zSRD?e=97fdIOl zHzVYQL-EQYE~`mJb&k2L*-pI91@Col<$V^zWrAeCE_i^;eoijQHX8I&f` zGmB9?p#uyl{!yA5I%T3K%B+l{kTkBp6n5caaCb;%e!~}wZb~g-Q$E0uMl64b4duIAUcnjMIJ(`(eGBeaItPssyOz<=Fe+#Fr_0rxW~ic6s`2uWTcYg4 zzvlaYuUv#E;@_fp)V}$OC|-Q%;~+=@Iy&ET)uQsI!}{uAQOD;uk%p+-RI!`_wc8-l zb$g{tmQuf8Ho0<}RC(h4w0HFBg@(QL{i;CLjpr!^?bDRZwm;s7X@Q`_m4O&M%HU16 zXWzX`r)P07=7()8+WUh&7IEj!*1J(jl{$g{vGess^p%O_N^@i3^k_x&UCU(1 z$1OU5lB|@j(7|&<6xHeikfg*62)X;vb-iBugIXGh6Zs}wNllGjnX~xM&xU;u{6SSk zrR>tnZpP=QsB~mMdMMxuB2^eE?I6)yv zvo2iRjg1rb8dV*qcS~k$@#AE7qjuYh*l@U@F?qDGvfT4d%-~3%kmO(+sf2@LxoIt< zgTDO;{W;Y}8Xk71{Z9qZ(HhTYoRS*)xKkyprVgac<2#Rf(cFluXIV zfj0nho;{^yA-_{QzyGOo_>J3#s}&&`nE@r_UQ;;B;}%&#dz!JCiM)bO2Z^3PDKsdV z-fj(lW5FmXi9C4NElGIVVv$&M<}a}L{V&ed$Hxb)+wf88pKzf7=_NDy(6LRViVI+1 zU_e4XRJ&gv!QYX)@lNuAV0uZ^ci?S0%hsu&@P!kyE=TM;5TPhDeiB5pLaG@$TNK%lLu;$P7AR zfp_Y7$WXZjWxi{R1W$9tA8M7?**vzM!P0fWGoHfliayB@#o@C4l#dLZEhX3y*@ zDt9T2Z?-gRRL6MD#`-U;mZ)k4&OdZ!e9Ck348yX}opd*FF2n}r;59O=aqeHt zqsEELXC!dWkRaOVzWo#b+Old7xdF7n{q}h#M9*=mYx$A!Z2#hvV#waY3y)#Q6~Q7P z-~s87efQO(U#Ge2(7(QCi>0^wGU2+7s!1Sd{HXpeAuwO$@z@{>aeSXHmQ3~K3h=t; zF}U|5tT*rL<|YXA?JnNw7z?mqeXT~@eiZ~Te7N-Z;Qw6V&DQI?eoNbPV7G{JU*Km& zU~#aQV&3!P`QY+DnQM5^tNL&I*(yEWMq1j^6M@Iq69HrL54z(w)^Rr%up950br~|+ z$HsvkDxLiIcdvUMetEn!H>`QOGFYI4aC74Wj(V*`{lm`MaVPL%>t9;cjQmawGjAMH zU*=c{syWVosPHx6y%zQSa1w2eta5j8{v`LH7cpK!Pxn`|E@S%*)>xqH;ljt?wfJqq zViSXPyzj*L#o>oLQlr##xF`VDOsF88p4)-xcU#ghf_@f<>T2K7EDBH0t?4 z-8KJ3L!-AYIdrpD*cxrfu1?Dd?rZ;{k(`<*!#GVxXgiiK#>;JCUnq}b?mjYX=SCD%1_xZIaqUD#)Z!ES(HpyEh|j)_VHZWKu;bI%~CTifHHk-+c;5* z$VPL^^e7O<`05q)=u%z;tC9}reR#8$D?ip<*3ZZhe4IY$0tl57br7|ll{q>EQhk>39 zAqlN?9sGh&S4~ncRmfpx8jL>LVO+xJ4VMp>8OUr8^RGr1CYeJ!KrRwU>FR#(ZogIF zjsMy`+^h5PUpaAY(|9(-YIvIOZ?2HXzpshicRsnnk_7Q@-`bwZb~(Clh-d|mc*xck z^Dl-7p66k!@YAaLe6O3A0|{p%-_%~-j=rb$y%h>Z3z1{#{@WZ_FYGq|XGGY3xh?3j z+1wUzav+d*Q$%HcyYu=R;QIOGyNLSIHb&sLaJ%k{QggG;sigYAV~lHL?Dn$&BJfTi z!7ugpzTj%rHSpsE#RKugO=;W>^0&|1cgGUPU+43>n;$%f^)H>&1D9L&Z)_>ymuLlk zh^vJ6^?u$#=Q0I1vuE!2&YQsx9;biu%*F1$7F>1|S*I8TA0VSmOJ|RDB7;FY69r-} zOLWK>A|`tz#Q$;7WzcrHHG81j#k?!0`th#ovzf<%G_z^IWi{g9ch@$ew?@AT?D;Cj zt5XJF-B<}erYb4YJ)Z5hWPb-Fh>ve^0@RhZT6%J_v0zGv08oLFL;jLp2YW3u4KX`; z?JeWyrn)6%tXoDWS@imv6Lz~rWfgM9kQ|xVdJ0_D0^YdN=#NIA)#j;s?NvwqfnfvBPMr$w-lWEPWAp2IZr=t|_5jKXg|hPuJvrz>(5*2U8DZzS?;6*Q zUi|kTCL=ntC0NoXOf+J2DPm>B`cHFuHX9*0q_Qvx>R#Kz9b+V{>6Om%28Hb=Qn05w zsxlMR&G}ck;SV#YLhafuF&WE^X+S>0sBPM)A;8GEUH}s_+}O^O^RorGfdD>J997Hs zi5Ux1Ll&gqpk6d4g=S6EpLs}~)1?c*qjzV9LNReQt;Y&W{xVF@i#DzZ*%pglk1@L7 z6Nn)C`h>`SuyL4tfs$jh{#M)GwEP>CJh}-OocVL6@Irr-IW0zRuIA<&JJ*QM->@HA zuX#FXhs@H>i?a#cVdHFOyPzmV>@Yc}&Pd<}Nh+OGt$uxE>rAq7z6~IKj*#mGAs~uc zy#RTfc`%D8j6)!57lj6uTzm~km>+qFpZ!*OOG8a9isYmSBpyzx7wf+JK~GL>MqYh9 zv2*p!^_e1>54;=`F*_@nc-&e(_JgmZ7f4N;`<)@oJx=%E7~>bV<#nw&QA+fzKlj*& zB~7*n1aFUF1q6Tqlj90K?(ulz4iCMXIktcF?K1}hwgKY@f-w|_BjL+ZPZJ|oQAj*dUX zzw$d<0|`IHo+mh`OeeSb*_#{ z47j(xTRjyT+RRs%RYq_9{%XB>cGE=1x9J9!BVqiX91Z)*Bu_t$aFkeN6MP21t!=^#X^)KN@gY=9thTT4XZ6CjKd8n1CfgoFe=6A| zEV_|M{lWyuV<$03(f>@@eUDVYLwS+<6o?i&7qfwif`WovH;*)$o(uj_5Ixzu2ctR z3YLN1K$Lu~_$E@ns|kQNvgDRlGQG-vR5@#iEHQpl?0CK5#;0 z?WPeiei7Vp%7O^`^IGD*2`l9Do516kt9Or0w85U*y?m~oqh!5qHAKB{nhSlS6U7j&;#TECb-N;V6yd3lmS^)&OJof!W> zRgc9pL9qWbLEhu&YTH`ibK|KsoL}Vd>n-o~#rVxu{LR~f2I)vUOke{KGr{Wg7dT-p zlC%(Tac<@O53blf+qHEb!@q1>*gd!AcDaWZ4?pV_?v zDoPTIa4+-|_$gaL^=l6O`P}Q|Jw!cGQA&!~c?>@^pf9V+G5!<)voS~klM^E-W$Hqk$D3@Q#ii>fTcYdhe2X2*3mK(M`31yV+=o|LV6mC^0kEvgDf`$W}rk@fPO5}tc z<@iL>`Az`L;8RR_NusfH(YxNhkVS}4qG4cJ^#(Zw{S7r}lxcv+7EgoLO{o~*n3C&2 z!miql@7@^*gN)>Unll>j(9saPT(C;WTZ+^n;kVvW>2aGGalHZnCWOxv3<9@j&o`aw z-g)~kI;}k&9n*T8yf^O{L-YJ1b{Zl8>;9Yk?B}&+HgNt@c%RJN=+RZ-(MJf-mu>P! z@G<%Z`O8N8kglh+3(v#u(U6=W^O-&VaQ=!^Pm3PG%I(SkG&rDFz~ocFA=UL`s+C|AI}1V zt?c{qGLGcyNnxPl=aDN!%j3f=*Szu(w!?!^Kwpb--R=qV&+`>agEkaHkd>)!@W|OA zu<&B4ke)IENQe+jmIcd|q>A2@dLKzZ9VQ?Q*?T;@IzBz)KmY zACqNeBxt|##e>)qC?a1$X!+;vWb|XxESK!a0d`i@qGeoa>M(?lb4;o1aRf#TW%Ekp zfq)5{s_aRP9)c0PgRBB7pf4Q?YI;FTVmP4o~Jk#c%v5W9OwB3OxRLiDF$A0B?vCP+8Je;@ zUbcJn8~g&>3bR=~f}8~Z00Y9C^(vay|1BJgM}{?LGv)=2uab1(hk8W!M59@EI{zM@ zEOtMdKz*MlClLJjDe{pytKr89rqJep5G1oUf$rRM%Z!x#k$}Y?5aSHjUNsn1 ziKXrLQam4?FE4onvi$E|^;_m6M83Rhx*#Z7?yUDZ+bA`i@m1G7zdR3GBq>WLsJHTAhczvF5VfHV@ZVqIw(AB6J@TjR=q{R<@uAUP_C zka$XL90Cptt72BNw8dt1tdwi@=-Rbm_prXZlUgFe-<|q2Yrd;Q``y|l9tvdxrQ(Qa z{w(aOMhT1l4;XONO;G+{weOrxQen^PhZr9{hZbY>rN6ddZ6{vi@Nncp(q!fSa!mIp zbwTt*6&JGLQt)y{2U%d05HtEOLUme0BJkbi9}@OjmPx&~hYe(oTxRE*Q-Q$E^_nEZ z+ov0i9`9%M`r9i8z!kD~?kVZf_3@!sEypK(Y%K7v8`;_;GTQw#TqW4!d^O?1=VOVB zm;WLPR+oKD{|{MPa&wwC^MP}P-}uhMm%)QbB266VR*{!4;(eqFX#ab*4Nm*niHZaS z;`GJ30s<~~V{dcJf_7v`*FIoXwyiIz1hExG8zBLa5wv;2xBPQIlHaIlP$|ijQFRgQ zz4P`Au)f0kz-Ke$u4vWU!Um{9wv}SO_WBjCIdrxnuoqVd5p5rRR7(!KgfdM-4)u8r zDawDZwPk!G3E_wlrjCizXUYK5sID*Qs>T92hk0_ifap~s0yBggqO*;fW^+5IOc`}aP-DKmz=88pcA@tT!a%5BRj2{r#B3Bg z5TvuctgRAN+M+xeCsdmmRjbNcdtuU2!#6i2OD`NMsY~w*RXY*ikGg4?YFWI=^hO}iT>vKY*gqd0hnJg?R3975h7oLU5GqaIeH`=fay4c-7adGiBgXUmk2(-X zO%vkJE1TN{=AXuIovs4sYxr}1nhesPx&^%^;8sB!SqXJgWKB`Ezs_PsRzMKX`{6X&(d zhpnC6W&+o~cpv-8?=An+9aWtKB)K1lqw~pWk^YlBF-6_xjsYRi=B9jrY+2lON7W!U z6{EB}=2j+1QV$4)B3!zr4i=kc2JdSXWs$?g6(_b0Ra6BeGJZL*V&BUuZmHb;^CmgG z&{Y+CqePTt?*lM#qlC;aH!2oBdMvwFq~QWp9Q8P~5#N{6$~px0`MVlh6CndcMM)mD z8r}xFE5AhzQFpwKKchxvN(1K%H^iJ}D{fre7hwxw9{ugR>l4X=SFS;y`wC8x0a+V`?q>$l94U8W;7qQnA#g)dFHF%hqjb^ygne3h0k)Rn8uLK zC|Bwh5)EWjoBqd(9tr+T#(mW-9f?nA>ve^UH$@KHym|0rKNfA5{Zw&q^4O1~-i>CN zHuJCZ!GY^)v^S>=I+u@SRT4JGZ!-$+z8%`RcHL7w-=C8`++AV4>bPyb&h2J2?^P!h zV$1g)d1XX3cl+D!t~c)_?wYs?{(mIsy}8+;$i(f|L?-~SOncWDx2l_a>Ok}F*`Sa4 zt8)~EIn^A;1oGni`tfsr#|x^q$GvYf*KitVOV_3$b(w^f&i45#Ro{PNU{*R2#F+fZ zIo~D)c;1937SJd3S+N1Zs^nBm zxc#~zm}*8EJE>x~Nhs-3a~3F_?_aRJs;+=UA8S?j6b?gBphe^iRoNL97O}F`JfF+) z`Ay>4{OW;hFE&uS?g-} zJ;y|2SB71;x}B*EfXnM3*XnyBA}O5pOL=_x)(FCk5EN9$S~FQX#^znm*XpqV?2yDv z^1^~>D`*HdjCjmGG8ZS>6tQ3rr`6GIkY)6|>kv9+P14LB)g3NQ6LlA1GQtFJ$Y}EL z=<*;7EryXlBO}6jr)G0JUm4vTvaP#0etBRJ4-EH7Ht}Vavw@YclMLMB?-gn_-wzC( zZT}(=-YrCeFod89Aw=$rw_P81U|G*FG_Ma&3r8jg{wtB*2;{Tp_v?ayu_d;4XPuy* z2m5E(s(4#E-t>TzKBQ_&c#{F@jk7m?-ELP?^H z&(Zh^gwiLZp3E53>+vQTL>QKt`1*obe4_iBFbo>P4h_Smg!-U)z)Fo}YEwKEDNz}6 zj5K#~tmRwuV=_DQ_(Q7Mo2+&W^CQu#`|nr!7$c0kJO0|{(A8wqp>orQMneghZU3t# zWqT$0Y5bCe%T^vXWkPI_3XL*dIVAg88ikkl-B%Cu*n;B7Nw6x7O`Dt^8hy7ea_BX8 zm>v<)=+QvS{;Mf->V*w|>=HU8LOPb!^+WF;mOl!=49J4slaxaAC@X>LXs`LHz>v#qSCEX#9UMpJ+Vn6p!K zsAmrIPE=n=2SSlVQm0Q?=l}Su7x!mr$=qWKXd{Jm+;L5r)z9yHeLFl=ugKdLlcXZI z*CNgaVQE1e@n6pG;g2YZDqf&*^GND>=)E5pMhYQYSG|_sx&L7DKWG^FM}e0MbmKfb zkiEZ}q&>PKDu6$yc?Y{`x&EwMnaSriIcu3e4q90B_5p+}HJSS!%m@3+hC{G!J;seiMoNN_x__1bc0ex!!)xWHa&|$RJpA zIo9AL1ladn5EY1difc4bJkK)dS85hXM~)Ojc3a_AORl}wn}Klm%9=8Qrz)P-CA7Xt zSTkSX<$%hwHm8={ZtXqZ0!mX2Q^?Hzk8GrMARb1cl=G6ca-xy1WCM*-ZZifLnpIh? zta(W<`>lAm|=Mp|K_y9EF|EKxj~vM-X+ zGOMHanv?$W*v5ub{U;eQiV2|iVZ)~2MWf?>?I}}^iwQ;zWFSvADoUaUlRyA^T4_G1 zz<}DAXmgazkH(j`q{o3Av#4k(KB^xQ;#xt`=%E$wtgoLu2h9Hde=R`s z-#EYU#V8U}Ls87MqN&H5+#k;Bo}Wkph$J7LdcyWiYV`5(OUpceP~-djrXSh*oX4k;EU*Xs?HZ=k!|D)GI6$q+U}`W zfWF*Fv{QqVR-Lu$!~5^VCcRanHuu*R)Mw@yU(k`x@#KW32e@2D?$ca%0ncF-eDk>k zT)EVA92YIF6GHXxJJ-3|LJC#xy0trwEd@h&O?PKpIPq-pZTX{*xF?t zV$BNBHxW1yiFlq*seb2cY!~1tU;A(9a1#`E@wmXE9}&NICM8eKKAYE$^2mj7dQAC_5p!dB8Kw( zcZX&!wwz4t%wIdj0om|&Zq1W^g!u8N5&8M}x_2aJuTX(Jg!8{=5Lwadj3lb`=TTDM zx0S?{e-rw5-5)4du}B?Yfv-bD&}AR%Nk6SSN)6)S&;Tbg#r!{76sDIWSw|$dT*@Sc zRz~+?hI!6`P>09>=M|AFo~?x>D>zV1ylkZ=>z@RH zH)l3Fdj>YR8Wc|f@O$xwQ9$8s@`TxA3ej^Qe7d>&`Lmp69Sz*)WcYNY{t>pQ!I4_+ zdViJB1>Sl5(s7|85!|((wtD+ltfGi2^LnKle$xN;T@ND~pwN!yti!4P2^Qqx!w$RA zjW_%^UnH-v!Wn+ls!{RqXO}o$b5i!+MsK@SG!68&J-LtM^?!DZLONpd$g{);X7~De ztICjGf?rEh&XnH_&Jb(K)DtEDvGqXHCkw{RIK!H^bXPTFG35jo)z2wI5ZmW=d@ zgo0hP#~1w?xpIcq`pfI9_zwGfOzUE9HzNRjqXcDGHKq>Onh{snZWwm!fk5py^bTki z8;_?#lX38{5G5$dQ%ySmAaLPlQI6@DIO7DXEp!A^st);2Y?)@iF>LRH#9U*O$^?Sw z-L)Od9U+XY=t{{EP?X6Wa?1;|A?~U$4hW!$obX-OlEL9TKZ%Q)&0k{B)!$6T3p#K$ zeMoeKs8d2*e4P|AuH2`j;JT8_=pahMSSy{JgPNS_R(afp+<5;SaBWdWxsJ?re%{=U*8|j$q4=} zP4QZGX=74Jrw+WB5?BjBnpBS;wzO#u`NRMro<99f6M{jAb|F)@r+f9@pa6D|u#VTy zgD=McPu{Np$ztNSm&jmHUVh`pO>pI$y22|Xd+np!-%%Oj53>CbQP*B8GZ88Pz|SDh ze>S@J`GuMPIEdw8M4qT%!uQXg3iBS%Qv!I?FVAc+k9gYuU!15Y{QgJlyU|!}E50TL ze@W~lCTYk3_IMqf`3A1_Ra-(xMj*nT-#8pC92~8xBnBBcZLKw$mGacpm<$amv^sIL zwAkwcTO^krm>xJ(&pOJ^9A0jWFA8NaD%~D^(3)2>)TEMCjNoDmGf*2N=Hkbr2oVBH zTQlp-CZkIYu7_u`8o#tGUmuXHYaL?FfuvM=2Ig;f+tv)dQQ8}8*?ebBY0^^MeFIa;SkdssR6Co?t>)L{FvQBd|F_b>MM&WB4^yY}^~FX^$+Wl9;P z`a~yWQNatPk~V;o9-r{~`c4SX>-+CtYuni)*i(QBF;IK^l&AXo0Qz##t%i|bLIO@l zD|73Gl)lGSJ8sxgv(V9)8}TG83v5SJqFG~7-?FH6s`4Kz_~2}8kRzy~1mepY?TM?* z%AN@QY3vyhgzHWmFjcZt50+q+QXOpGH6=alu3(ZlxIo;Df!wa{!pEcx0Y=zdF(Qc?{UN359Wtxk)XFsI-x%7idVB4rpqH^ORvBBQkE*3XF0 ziRE-D~+p2kz3d52jO4mMr zuBez^)v5UASADd6tZi6t8}KOaTpY!mZNIS{@lv`<=J8PDI&x^VJa?w4@W2H=$8?b1 z{nbb2z5FizE$8>8HG5+FV{#p6^f-haU9M%OgPjFOhZti!1QqWEhFu)S^yDf#4Z2ii zD(-i>Xs}bA$D0YEIoUz5nfM6O zu{a!L(%6=UxQiCmblM#(SQaH>oC3T)eHv0~^z?NV`|^6dl$&tgG0sKDm?4>n<-2l` z#?dEr?3q|K0nT*g{NPg~iPFjJLCOJCxb4t)Fc5v|nu3o58CV|8Ezi#L1VDu*jDx)5 zr+HnzGO=s0%_+;|(;sG143W9tW77c8i4m%wd^AG=BDAxchQ`Lq4vrgsV@BbxANAL1 z6BXXrWz$Jo*rW4X$&7AV`31BQ$7(kgZV_NaXt7D;6I+6S%>7qatqrSIS)q|?*~KAz zww@o{(p~U2J_+WFIWjKAUK#SW*8RyvwMB23%gE`#yiHE)5kU1!{ay=Pw=v52OSCfGY)U)MqQ>DQ?IGzHCG8yJ zOK7HTCdK0FfSKV}thnwm2X?jNSMAfETa$tz&1T~=u-qyjg)ab&?qncjyD(=V4@diUX< z^FAJFJbilzHtf2umeq9JkTMs4KOL*rxl5GsprWDe`^)?IHzok^82s#iya(rKnF>X^ z9cqq)H)j~ugqz;gW$^TvG4M~x5~=Yv2oI+1tsS>}AC%_*EHpZErU5`t@SKc0dUxWGlmlV58RYv0(sE4;Bl$jIpDTZ>z*hMgWOH zgxNDUHm$)%2&Wd{^VN#SsgylSOF?N9*rMlEOHO z%ZmI-N5oSNxLcqV?TV4_cX6;?UuYXMB-tRS`cX8=DMNXf)ushW?Rao-ldNzCDvj%g zFa6RZ@Bn7tHiH%N$6tfh7i?w?`ub99^aDnDK)kB&OZddGeILele~CrneFqN~wooqf zh~vpPahf>QxjCbt8=?;yzo6r=3}o|R7Cv*Vb@n%K7ZCb2t++s6EdNb|p>;7uMCj8H zKIsUSlmKcbQhbl*gP`O!d=<=SXw^M8B4if}8D>)RMu?;5>zd_ovxWHz}5{~CU z$ZfA#$(NZgbbz?!>y>l3J~|-eXI+8VIl<%p2+Iz>7*j>dMH>I)OU`T4uy|xz1~}-? zy!le_PJ7+UyRF;#6;A{jftcv=+qTCG6LZlAwM_xB6DMy@@8^`dfHZLB@0Tp*AKN8r zec?tU?`~&$*Rz#*;Jk}#l&0;LF1XeF5fnN0|BgCC{4mz6?q64VRLB_d^~vZV2tYMo zk5Z!h22rqju;SO#PFK9x>D+ca0&g4ngXQU&fD%H7hrPzNGQPy(`QPXNvRn|EU))t{ zO4*#~;usu6bp?h$YDR~-zxdk~g#!S_H4F>y*M`s)9lxFUZl6VE%mz9gd+>ak*Fdg!;XDdb8~Vr zr12KRctd@v zV`g|U7b?mG16>i zlmB)Pt)N?F(!MC^z|grRLk2)Buq8dZtg%5qx2Z~L?~@&#uq{H=Tw2jg6Ad+K9 zth{RY`0~*v%k+gL=)E}hHaq=?QZ^e6&~tn&a}oCv>x)H@cF$f+o(J8Ahli>oqn*cw zAph_dGsn1gY8}WHfE`AM{he1S(97L40gXh^YCmns;u?-oBn)aR+kr1$ji?_B z-X*pLt*Wg##adr?P3$HxF}()DNN%+iBa?VH2@QpZ&*N*YevKy$e2!IFlR}z-c;P z$75)t9s^LAXaoJ)OKkhgCT3mMF@%)5>9a z{tMaUnov;}M6Oj1(~jUiGBLqpZi!bd%ZM$F$%ALPmJ2RqSHh~R%%sE)cF1OS`wmO{ zlhm$%y#Efu&C9#8Mn-`D%{Xr#hf%*4LsApHnu30DP&nGS&TGYO1nJ8Ps_)2Lxs*6N zf1xhC#c87Ym!0fa9+AM>=GTefkAR{(3WU2i5}P*r9=~pam?|L|G2D>cX(1!Q&Zr0ZWe&+w`;b)71{{%a@(S( z-2KA1gxUS?2dw`9g)2pf1OA=RwUzP4{Le!6b>6}j6A-c0L#vVIgu|7;A#FW~R4e#t zQFz;uRfSsXs^w;#9me}fM$e~JrgzDBSJw?GU%q^4h2)_d;x!XDzt@SGX5>@G8h3}^ zzH!80K922^O|{o)(D4 z2lf#lh1mCBxeL*uCXxicPJd%Wi1B9uV)dYpA&gyu9aU@fwhe?*N6Z!pcn_473h4>` zWjJE`!HK4KO0X`i=|4@3_vnhzlYR&ip@$(!3hnCfD5A!8<~%6(Xt#o>JFmHmSU#}^3Pd3^#E*d z`zg9nj0_X(Jh&SfUf&|Mva-NY9{~$<(DoX(G;w0EUxF;TtYl&;ZV4+pab}=g>Zd9x zTVl64RFT#>ip|%v^fm47urNAt0u~-<+!Hx#+TyT2$tt$w09`srqvUR0%q@J^Dgy}W zzwSw4nKA_Ag~Oa}>d2UE%681^850ND!0d8nt(%)MiHc4xc|27p0H?aIy6kC5fBv+; zdeKC)OKbei)85y$tZhQSntCLcQtJJ8+c0&aedZ^m<^My|7(X&$?-rKDV=e;`GAOtk za=4#3SiG9ld2_qHs=={WKRnx?k>7P*fuQN$jP=e(QXa3XLxm`+3;`jPy7sxJOWuzg z0tBbuP(6;bkV(Z%h^o}ypyx1JafX=H5`^#JNqn&8ZcI>WzTcqt7ja95=YbTm1Mg9RS3yEkgMr>=zTE`5D-R*> z<{g!;|8Ml)ue3wN#H>)(H6%<|R;Z}Bffg|W83n-?kTSjF;{PG)Exe+7->B~)9qH~I zkWP{AF6ojk0bvLM>F%MWyIUBLZcwB9l41}U+8j~mzu_Z|g3Hczf@e=jx2J)3NS%nL^89#a z#ew5(hxOFsngAI^lehDdLSj6+Y{bT$gG8Y~WiS*{NwcWPrZ+~noksa%+>Bsbv<@*j z3@S6wXXZ8c9moBn7qU+l0vqNYpei+C$QDqb0FcRNekCBeg}ooV{D{OpDQu}PSA_GG zyyk=@1BCcW13*;MIp~vKukVPgtO5YUhL&{K>l>xRq!vd!QUFj4a+y+y6h#FH2r;$p zToGNBkNeI5^~cZn-v6(onMD6B6CFr}T|)|wpjd`c0|_D|W!{*FMgfm;E_hv%dWjXP z9g@M}15*IM8Ha=;hm)~D)2t3^8mo9W=FyQ^h7&@7#Pmt}qviG{)%6~b%E$bV=%~oL z!F_8&zio)bmbZ+H+{}vs5YkKUZ~Z#rz#iq8Mz4^N;7lYW;$$2Yjoa_mDdloX07odV zs|gev7cnJaFMtv6`%FZAto`WZ>)2-fOr(AzzHjO#{^8dum>=??vgGC|dNp-!Ycb!s z`KLzf8n~_;Gs1+1CL1*VjSOdZiLATJ8iB!34=ya^veZP+2H0EoLX-nWHEk)SZG;{D zP7N{s%`!$Xvg)(xLI)8{Aa2f+0LvLA7jrS~`oY~-K76tj(H@wv6@<@Wce>WDA^CKz zrAW>1ve9K={N60!GP|U8^-)5y) zpsbbjEPkzng@ z0m&HGB0_zLsapn(Ng%BIDbh(*(O9)gTzcKenW2m@1qVlOZk~?Sgq%6frxs(^vWu>- zZ|xKGvM1kHTwt)&p6xe#UqBvlj0_{~SF9eF;UjK5Eh#)L*7z8O+czN)?mYB;E4)-= zqosDY_Eg3TDRXngm*4GDy%xeQM=k`28~~Kv24$j(W{u>ErP#HP8#GDLUnAz5UFEU_ z0*JMz5(_{jsky4&KHVyXvMtwL;M8ID@kVwYBt#5QZu63kI zjEo8rG7k!745~^93bTciJ`5X>|G{+<8GxJ&g6cxG)wV@!BR2Et33{d9e(P8jKKK)N zEJ)B-X;vMlB|YN7OTP1KJeY1fliHH&W@bflP2)(kw@X-^T_vT>h}^8+@qU)n+@$BD z^2_DM=9>X1QMu!4Gx2IAnSskVH#|F^tt*%)(K*Yem#*;;dDYoPB~sVz)dH+Jb$=q!9h8<)D;8S3F0(~ zc061kGxzcS$8gN^7IDDBPkPEBt6ZKidjb))8rk!K-HL+1j+e&Ih)__^iq+*^vDl zJe>>}oQi{QQ+i&MhymM@LARtDy&GB3my@q95LVL{^E#1J&bfW6z@N0{$bjG*>V)oN zZcN(y3%d=03-g-uh0^ZCUx*j>|54GKfZ*}Ho6|`8l6Tv1wT1RFM($wTVa+& z0)pTM z-?GpCZY|S`Yyd%3;VnY3Btri_`L@xabQKg6JcXqo1<+M_?T&|P#ENIpQlwvIrG?Kc zN*pnPpVd*)e#awiYROd8@{X zw?(%}5HVTCkB=mce{mrVdT$hAS=r2_9UsYrBt70y+%+FTWT~$t54(SG9d0O=&upx- z!3Tt+l!*<23`vIe025kaHIesFu4`?_xg;}(g;cf3lLczIH{+gfd251&TZK7 zai-WN)|}$-s5${_Q9;2WO@)GaQrIWJdl$BZIm{ zc{!AXHw)#uh-14hkOS(8alucGZ%Q^v)MP;`V3{6p-7$1p^nvxbz=b4gnEi zubT)n_4z$^Cj7GwZOlb@=ceNH<@o?-#nvAc&;dTW&G))0nA|Y4a~u*3YW(OS%qNiO z!zurAz5HqU?~eIeMbjv`>&mlv$JWTIM*h!WLDDL?K(kc^LlISWfm6z z*zuIS@Wbu>{Ng1k(F)(JJzd!K3i=!DslmcQjS4ticJsX_t~INKpA?AN7zg=nVM_-2 zPwC78JPjL5-ksLaD&KV^ z8FQ0&+;CsWdMzNP$-6U+95mJ(Itwljk_iuFp-xL_oDmo0;BiYlE@e z<74>%|1S$L_zDTpex^hg8G-{~NeTaGM0U~q9+aE${$AHwSH;esBngHQF1>_l*FY}c zqz{q;pfVztrqzwR!H`uchU}9|ne2&lg*_eBOzH#@XpkliF~>W8M0*o)HfU?@QS0Bs z7k+CBwSMy9jLpEOFC3LgzEV}Ql?Il4|KJ#1EK`WWU09`8t!z?8dk}9AV$ut zs$yFJ89(l_t&payPmI86QosktDeD9Q`(3euRqRPF`lzPX=?ZxTly$(+GaGjLcqBGt z)Lh$P?5m~Whyi4Q*M+$h+uYMA+UamxxMK3$9~L=RqNE>_xHQ=}=$6ck9$n!6? z!VABkt-A?@buG%uZa0N5Z6nf(&O0tWpKqsK4%(v z;}d-9aUcbO5wLb21wXlW{_oy@H`D(OFBn#v@K3Pmy>P_kn=fDRlYfox?V$9Ncn9F9F@?p&> z89@TZ?+g<lCim}O8a+Kis`g}?U6g>hU60#gN z0RRs|@*0)sJG$m1tM#!`8n|A__S1jkkIlQ75wB&NjF)ejxfz=mPm=KA1gONnKjNMG zUVUs+!XIfjoj+PLRGXGOWG)uw+8da>gc(rxu3mHE!XIe$yS06;zk&DTyH5SU@mTH7 z8srmV52{g~wvtUwUb`2srnq`LViVI()H{Bkr=cTWCa3;eX-`(Z$GMyVyd+rMv}V@I z(qJisoy&;>e{jh+seh~&b?!KY8<@JUpV4@?1TBY23h(?j-18{T|16vNpD2mY$Mtx> zH?=w)XO>ObGLN6!Qi%HKk8QquG0!eN+Uw$U+B@0kIa*B=t8|@`oIR)|e;z3L^LRXp zb?5R&^4TQly!*8Pn@sWYHhAr$n_F2mmM+WnQjBd& z*%jnGKxEpMZFh}gDVKLWc7Nwt-%u+^$y1Yr@_-iR<_J-)$Kln};6*5sc++1cKAv z2-fW~kmY*(zBM*`yC?ch>|`$p3y~PliTHg+ga>v~9})u3GS)Kx%f>CV*L5|ve)KHX z$dbki9~50joln$3sdu;u3{v6+E&Q(^TytoFJc3w#d{qI@6ugXOXzzlpU;v7cV159(YSqAed2_a!j=kAi!TKA` z>IDS5%-j5ZAOhCc6=`*;hsPbrp^C(%wtAdaBTPmljxFv{5pR$&x$}2Jo*)fmlks#v zYrXN#X$aTcQ-c-&_;UJK|KbJbELiE6ypB-Sc!2HH`T0*~letkBXvc59R3Dt4atB8S z1&kVcVRad;VIDOo6Po`?7ut59kG5XIuZ=mZEdi*RQ^#BnZExvKoKP zkngSAxPtzq_4nF+(4~M9X=Zo}-v)T?9`sf|w=CXJ@|rQst}IAcuP&>}9M~*YabWOZ zyu-)E!NJ18`BcWFo?96PkV3Nz!A}Kk9%0cXP(w6fkW}~=D!((m>~bbGCx`GngU^~I z$&Mje(5qXPG60g+-bZADBm>(Iwt(qU;Oh@dB9=6Q&X42dIbR|MlvqFu%ggkGX&CZy znku*k?6DbKmhxoi>TQUv_D4D3kA(PUmO^5Ddiv;KoY2=P%m`dW^_mfS$X8i8G@#t@ zIE)v7aYw_22VjpgjBl-}nMR*B;)meMS*3H37=Be^g*L%ds(G-UGF4b78$y;FnPHhV zRi^zS8Gq7p2|c$RO%QW2c73DaDp4q;R5`glje?)c)N` zNbXHv+Q|W(&luUq&Y4EBzP{jhh^`iv7{D|c!#JH4qOQy;g~_!$z5kytR3EAviH^SG z8(?8!aX*V-m~j0uz0`&psU`Ot9@$7rP*x-@?kvUshiP&AE|5Rya2O}#b>Ar61RaMs`z{&0NW&A6M)#Ujj8LhhOL{r9vcCCbS9;brcz8)b zq_5wRH6a+%NQLgZxmF}8I5)R%l($RCW7hgtej>ccw*T_C2%b2?3hF{!ml~oc?`Uan zR*H;Q8n;XS@+EScKHbWVT=l*@fdD|yr=_2K`YDq?N0b&2Wwixumk?MN{v;B+ob7XNZ0MO$D3P zrJks|mlfvC)}iz{Uy-r0$!I>WCHNBe%XyDY^(iL*g*d5cm-*?g;g^Atj9vnEV#FtI z7=l#EfZoXw)!vo`-hqus)#Xwk==N z#e}ZS%rbtt-QDkV?EqNW5}YSBKr9!VK$ zZC*7p(d;l`@$~XYE0(EDo0Ya2pX2W0F@XZB3h@vs-r~_ej#s2fUvqhnb3wC4_RuG* zo{|#l$z0(a$F~eIGFAms$qS(s_M^FF-@RYrnAPxJhxiMq?hhi*8~Ih^g6zw#a!Khl z4l0AuAgS6a?AQ|78IWb+qufR_-0jRXZHYQ&_$%wLrY6<;kx}UIhkm0MIaC0NU7nNz zoA5z6J+37B**oxOjBR*NV>ZwEV`}gFakUTFT(9^9Dw3>HjH4d+g>3^Vt|jNTfM z1nc*Y50ZH<3t=7SqPfOQlT#Y7%RaHEcVBPE?2R4$>);p z+rOWaOYSe^L^NaBfiJQ6)TU0O}P>bHu>=Yfu5}Ht*h7ZZF73e?mK3!i{F0qs0_I8BLm!s*e3JuZ*4v;m)ZtU zG|roC{QKQ4@foJNoUCv4#)~Gf>0oj zA`Js^`4<4GQQ1R6S+L}ehtEGZYsVIUdQ=dX#>1eqmyhY=iHkui1HKLbIMD#G!b>6H z@k@4heJ7Vn8t=C?Hg{b_%SCBa$552O&y}&+{Lk*XRX2M;b(Bnh}_Vuw8Hmn_T_@Zm! zvzix|LPF{0LvA*z>3}R6+&2oG04e!kIu=;RyXfWRQj*2UsBgby)3)frY6=dY%kdnV z;_3;_=T6_qVi}UDGdPK;exD=^l9FSUmy;-Bs$(-I*TgoQNX@_tvC#MC0;&Abt~&N1 z0XeXe@UynSnD2Pt!FKqr*khT>;IyKihhe2=ZGPoGgshn7I+z)UJrbCLg7@|PRT{jv zzkV!eyu|Zy5asLU!m>U*xHN9-a-Ut217CRPzc)#B8A8t&9-9<~qR=WjLDk~5FGSv@=!g8%6t}USpdNd#K>n>H5J5&ce5;pj2iyqIFwEk`Cn2}sWv!2H``nM0s*}G zkJ^d=SCe*jbU63i?<1PUR=K&EXhl71UUK+cxBCsQR`&8FE|LtN_lsRa zcQ0rQ?qH@G-JR@Ba6?_`dI@)T#={Bi~E$&2^`_u^n?9P z|H+>@@i#GtRZ#&uO(#G=A&(g+t+y$+73{}Xmta<;O7~knE}`jGy*FO@HxcHN4>>7b z%a32L0^xmsl@IUE45;DzvRpo2h$>2$s2)!!r&eZyxNT`4F3n!$Uclc}AJ;0@J)esv zJp980NFDq<2yp96=Wn?`>H>Zf{?DDgpr{|Rq~!PNLyx7tJu)1nt;ts|@^-efskIr}+}+)%OHmVt1<%~z!H8E5i6H+eZRQmZ z*=|R&v?WlWLq%f*#Ln~pE!M3n`QF4-#{QLs7){}qG+XVeRj;7RAYqV z5-Np_1H%|#QEDS2Ca>4zsAQq~7BANLIYftu2ty=pTV=v1sz--PDj2GUL^q!Vi{Z?9 z*F4#rK8j#-y8K~54&@y*VrPM{IR4T!AY=~h3$FZvcSwhax0xQ1xrb^eJOmuC0ltRF zQz(lYHen|z(3-><*^JUz&V-PlWe%w{w$`HDe*1-jf|zH@>)mm*q@AU$ zNT6X|_ml`ZL%vI!ZO7VQ2p=B_C=KiCA5X>CU|Z~>!)#0ms^yvTrg!A=0?8N=67aEu zu<_oWws!+k#tx04P2d<-xhj~OO2t{bw?juiQBB_Aw5U7hzqj8s7- zns3|DsOnNFUh!LpGlXw44LCidN=T1v{IiBEUwlyxm4a7o*65{BXJX}&$&48?%)e;j zKC8^E$CieIz{QTT*BzgpKI0bTaQWTA3O2prT|pd&d|Hcxt0z9c@uPZtw#TgYF^mHq zVp0NIh6XfHer(fH3HWVz(87PL3Y@(+%e@^cc)3ew)+a4_dpYmIE%vu1=j?v(WK-gE zD0(5Ui{#Z~ciqdhDP_>_T-6@W$5TpZ3%~HnHsPkk(FIsEQS{2XGtStWdGN(?l%HpJ zB{#X6d+oMUQv5CvjeXW?F%_W$9#F5|)JSk^stMh7@f~bDR{3H7_~8-d>|yE|bZ`@e zsPHPmUH3b!aX$8Ycs367z7Rfm`fKHHL{VdgbJPYC;sX229`^}+4eUhlQnR%$@+ z=dFkWNuG*{F5uq`#ZCPQIZb$wOYfD(bjnuC^BayzD|#Ypcs+_(jv(s;qs<%l=9)JOvujr`r$lA4dA?neB(^blz>kd$_!sB%0vk@4!7 z$ZUuE2*XBmUQq0D9&-v?xB8oI9HFsSnovT(EOtzA@Du^eh!g{#o?@tq zS0s*UYRP+hUReYSkrOl|QIzWw01g}NPbRUXF-KZ@Kq7uk`1wHzU&3-0N^mi%T)dF4oIdpJXz+dO-oKrLk0os9dH2sWR&09 zF z1^{8Dj!>RP_dz41`bjh+*0NM&l6nF35mXaIl1DkW5%fE*83Qu{l9ktpNN1QWMMxzR zM#Rgo5060+p^+Nm2f9x|Cr6wH$bFN%eDn`p4rj@YFpQ+zDOTv6DhOU`aUq5H+E|^eJAa_x#l5c^l4rveYyB z_p6AfY^>PGnQN(EZs^Bp#6z;mRYSzXh8r=80>H?-&Qr*U2T`SvOaa2s=BZKmYWdD)<&?lUnb^o2_GC| z23=3b=$&%AxbGTp!S`L=JI#9Nx5g7q9{-GOelgp@_|~;^kzszd@T>LH#O+X{XM9e2GtK$FOM>d1*SF@G zL+!wCqP{2rm4g3EbY?I>vYU?4V=4?djSL_OmnRDjNR8$(R&5{s z75!ldSeXP&Ve1l(bfV90A55%4i#Rd{-7JW?0%83G9nkZ7bu(6H2C_g$yVAJjD^kQ9P zdXSvt6`e{_Bdb;Up{1q9Awa9-Ml=SrNwT*mDV&Ce+;?2ln7@WZ2SwTBi%?=IY@Rfc zg~;t@cJoK)HNu3z0gg>}T>hw9(6?`wHYKMw3v}683{~n~Lfj^gzLZ)fU(Tb(VPuau zK=GuOTp0I#zbOFU2IGae?1jGPNqcP~7P37Ao>!-5r4HvTvl=}4>IS2*?)t@=uww+Z zh0aJ9{=(B^`g<|38~&k6_wRyHvv%~3h;bBf9A?CDjOX5`v2?)I-UY02_4e*escUkB zDeId~_6(_*zz&?_Tw(V^ev%oDFhVmU@I{~$66kUi1ei98z-Tv8g-1o18|BxM1v_+8 zq4os5_k+!Xvl?1yUjvfJLZp{?V+tYA0V?Lvj<4APAhpLS*CS}_WES5+B&BmE0zrb< z?sh{kpB@z*)jRHBd^b7#M}>0+d%|xy9u`LYXxs*LxMTMqw*KeYiuvt(eS|p60iDY^ z6Dz)O&d!Fkz2lNz`0%!(OV{C?;&JCHmrLLQT*7CDGpP40*YxRuul*>tpxZLqEA8)v z@iREzT5X$OtsOnm-?XBMd);%tCuc(9yeY`m9Z+~V+|1l#)2gxFFc002E<4d^zmwgd zip&YPV@_QA2gR-V=&mo}P_1DCPqF6n+jvx+?6UvEW+^<6SUQ$`g3t8|A3)ku6!@0j zZA}d2-c|=a?kpA<|8m5U;tFMx9n+`1+o#5)dAcWk&iZrivP)8McVnpN|7d*6;eD4~ zBIsAbOZ`9bte<7x;orXc(pO%~l=Rc8y$B7)F>{FMkYeM3C!p^U{?Zco-oSy~rTbvA z;`zFd`evCk4#a-?=Q_d#`{{g_*xX0YQQgynd4Fs7H6T;SeW#w#4+d?#HQGde1ziHQoM_O^B__cq(k(@sMkMGD zC{oeojgF+yj&tjjk)@zvbXdlSA684zMq7p5j=fHWlepjIv6_g;3QEWiWFJb>zD7)v z$DMXNQclxs5y5gWNAlt%fZhR$|C9X4-3};)9}N_3QD9+F&wlIkx2b9%oun*HV+`qx z2l*hyd**$ zw!zguX=A#-As+s_vpFg%GM&q(%8b0l{+s^>wiU_kuh7gYKMET?AJAJt86W@Pw;uz} zry-g6NQ^|E0ghRQ6ihX_q4w9Ndy>i85^Ej0T=H7duMiJ;V0z68hjOmRajvosF;OZk z%NZSzg!e<(0IEmIX|mMm?_*Le3Wln6FJm;Qd_lkNHd>y<-ydW`0({2{w(IFaAKHiN zI}<(P%I%KLQ>=f;7rAz$L3r}@kb0O1JB$%U_97>>h!skleN)MPjOpfrSI<|M^Vu-dEVN}j&A7xD+fl*s zuVRb=l3{g)g@?!c zDfsG61xb$RM-)`ite}@^+MeDeiu@iu@YFSl-SMxWwu5+yjM;DQCW}Y~cfYmFh1>nK zH9kPl*QNt57?}U%U+pBBzhvlF<^M_P>+ASuGq@s2DGJam{19zJT*)#WB7hR`#O@0h z$mZ!O`#=TStC0oC8#Sp$aE1XtN)Io~;2RduRO^w+@5aVx5``)2 z;9XSj(ZzkyoeqV1RJ=Jp{*=e-s%`9LcY8h3+(d(bH^NMF8-Og$#V!Mwn55XHufr|t z=P8;~y)6KEua()~fEJ7<>;UYHG6eV;MlAp-Q|lr19Fb;dx2gS`zCj9GeF%taJ{wW9 z%5B68Y?_Tpy*!p7Q^?n-KW@U2BSXsd2v@4i%!B}mgKwy0$k<32%joTl8}wNLBuEFD zjWl2IzfamkcTUzfFR0jjck%J~98qQPL>NHB2Wt4JHKG0?TJXaj?r?jen8+&%Vtt7M z=9CWa2H6mx&;d}%?Uhy71U~+#qd`{wkjUb!He1@5mabG83e+i^BIRM;PFp1x^BuW$ z3gg6BGv1Ot6rhYx|6|%B=&V88&JSE=!t#ML8=@(f+YBg7y;;!h3$5&07;V(`$;Ykb z(Ajx!zN1Xd%#i?X&Wx9WJ#Y!=s?f7+`VEY4@Z091rOz7@4%u3K`uuZyR8_Tl%n!w` z`*5gUUaqX!W>uS`UwONAC`$X45g}bDOKGrw$^Xj&(2eRTMWlzKh7chXU~a6x7^#8bWdnO-zkgysx*&OX;=i~c)H)FOP?1CyOSPd}bo zHoOpBIdw>kt>$;X?iB-FV-19K^iM_MI|iJ{!S>AN9)mR2`mLWA=Qe{L@Xa4~ywZ_} z86wTC%a9M{(z3Sqyl-KatByM#q_geFL?A4vAeC-!>Obu;a6GFD2X+MchVqmV} z+6}=FZ$t$6;Qu;4QuOZC{Cm>r`JYP;03dKt+sk&OlX;y^?R-oAi<5hpN1%!B220Bk z_vX%8^M}4oMWro$KTo6|<}TfQRo(R&1OskkyM~m#XP7U7u5*7PwDAE}>VaZzbAqpG z)A{k=W#3Yl?0D>McwmAbpu(uu>ev*C!9ftxMH7e03;)qJUd2KAri0`_(1EDaZl$cN+j8K-CJ{AYp zdFSt_4)3eP+B-WVzEm|+5rtZj$ym|12=MC%<17AcURr@oPFhu&IJg`c&t&KxP1b9d zrl}yd_m1@BjjHH?_^+ubz?cJlwtRwlVje2G9CTI=HC`x1JRyft#^3-!Qp`g-O(Q|2 zB&k>99CYCrZxwOFApEe8+H~4%blRZ8RDBh{yaY%olLt%yp-MKvY{xN^aer&6u*)YH z;_F4}W8HG9?KqUQu%@hf2jyztla{fB=2x5xumdzz_;Yo7AmSws4e7OM8E5%;^q?2x zCDL&}=V7o-=}fA;k}eBT6V6v9c9mqI1Z!{&=`f4DO3~R=%+3;pEChoXL{CSLW)xB6 zIl17luvH{&O)$ADJY8t}Xxq_6sg~)EmJl3_2apM)NTK@bRVEC66FV~l{Dj#4Y|OEV zu!14%LtRBV7R|3@qC#R#78k#-V*ifmTTrpvp0jbtD*=%)hyfrxjARy98Hj}9D^I8BPZy%fv@#nYe9QJGw%fRoFbD_&863FAR{jK1aKwI@3-;~=}$KEpY3c=TT z=taK<4MgbO*eJe}p|hBflh6A4+m?hW6}v75IqPm6L`{vB4cR162X#z ze)+)ijb&DgNDiM>UL_)8n1?hb!bUqhf=m z=?InvsF-#KO}`OnAyG1z(wC0Y2pZ>T<_GRYH@C zfk-K+x`jss1nJ2trWl9(h+_BqrT|7mhGlL&@1_0RaIhJsSgmvj%`HKa8&=t5ElE>= z-qM}1-uhC@4QnULqvEG3xZDw3L~>HtG#c5jG$dS#Ug0u3P={M0Nc_Sn<@U*+HFOwr3HB}@fy6^tk5H}mfrvCn zno3y^9MSmj-xd&3@JZ0+nS`Cq1FvN7!HTtQU9E~pZqHOmj!i|Gfun#A=u=SvB!@90aqvGso^7Dgj+`}?JHpaP(NKOEnh`!9WAB&yg@G0^!%KPc4i z@GoV7-^oY9iiP8mftz&Ok9G)Ic&7gaU1ew2aL-i-W5Trlrxk21A;EPn5TY)Ixk!5l7CnjzQ1n=4Q`~F+x>NsEDHrYvP|zF5JXS3HI!-ELFj*i0cV5 zPgt9!Qf5@m{Ev~EjGVl6M#0#*v6&YpyeKL3m611=CfP7NtcC0qyqnJ5UKD@#07m=VY*YlXoI zYJK$h)HAwHMmU6M%sE~%%3yRQ7#go>cDA=?O&gOYg8&J9zr`9Z@fshjR{5K}i{0tE zXTLQ;syY5U>hjT0K3>quCgClH2NzO*dPdn0h4sZNw-euho zI!JI(>`%_gm*M6Ge0Aic?Gvq@bQTnErg6>%r+LCOVRj z3h|j1ve=#}Kn2LwH?l3#pcsnN_@vBFJkfhA04?rMqw#OlV)$bULC9>TQsE(%qoOXt zPG`+Mn`ckfCs#Q8(fCQ1liww;#IAC0o4R2u6~|A)1q7iuQ6$I&*$z+5x;sTOY)q1u z!whw8+cE#O+x|EAAigzTq-eYmY8bW0yw-pE7?QX|+Cjj53AZUcmIWhL zJ|;+H*D%U6aEts|Y5k#*-8uJ3-Yc2Y`~?o4d(wAaY2)(ynEtmYXKy?2qjiCuqX`NC z@Tz;S@qns#n^d@*De)1CQXSjs|5Uyawm*E@{y5fJ5&Me9?{jCMD6CSeWKcG)m_>){ zvLj5Ci@~;LahDf}>fP6vQ6F~HbCdVe#7i)5;BUa~!_JHQIFLwp)ev^GJb&js=0|eW)@3tq-z?U-=L6R3 zY;Ib<^x1H3W?DU2(9y}$A56GX=TmWnqhpmxZf^4OiE{Kk`X^pFF2|(((wv-{xb^MM zYB%JURpi>Kc+7;rH}*00eV|9GdL#joBR1i{SrLj^oAI_eYkt zCl2b5Kt6?FsKD4Y`9aRjMw9KfjeFf1r6ZMUVv%^s(Hu6oZ|c?Jv`1nUkVcN%_+VKk zZ?gm`H~i$18Dk0f`L$+}+Bx(ztKz8iEIPNd*teB+&b;4mutnPw@}91I!K$Ys8TigJ zs%<5QB7^hGs?A;CG&8T${c1eVYiLZ zNu?v&?XdyD2Hod%FI#8a_t&iO7po1+=f4_3Hv#~%pslPrk#o{@PUC?7B38vRzYF1l z?y|=^!(i_G`+e2qxX0f*8loLnN}P+b{A2QBi1hJI-WmMl_0kbG?Mq69`lFyKZ%d77 z9uZMv_xW#u^0`g#Gr!um0D#vx*u2SHzn1|#U#Yc2FOJ67`E=8H+HS`V5K4R{Zj)@P zb|^+t|HUbAdL|{z`6au~-`H!FFItV;=Vo?cETEGU^nk)qud2T$9zjG2MGHpSTcj#=@6+Y+^68iC{G9YHQ%NF zDcGlxUTdJ~=W!;;(#@NfFLQupWFu`4%R(;mG9%CW^^A{V!ZPBp3rh&kIJRa{C-e31 zc3J`WzV=sYW!G0Z%p&0zWs_+WW9F6WY7tUqtIS!)Stu#lK;&=$y8Ph{WH=>Ctl9Le zQdA^w$%y6}c~;rI-mALM$j=Z-sbNP;AtTv4tjWmA8J(w9l|QjDIaoMmV;qs@f;H_c zNYaqM51z&%y83o^Q#3XjY{mxTDY?7O`(Ayp@jy_Q^1jR}8<_d*ebcL!xF6AwxNV<; zD#iSp^4)tdfBe4XMEaa)(DPCOII%{Q(X_-)N>s-Z9wqt@UQCJmo=jo>U>5ATY0Y?= zi&#bKOLKX8au+bYxnbVyy)?3`6+Uaq=sj&TCYZa}cYnEqC~8YR<4w5nY9ufAT)Rxd ze{1AFH$}gkn$70?F?bbzF9(;Vw!IE~tT*W0Ui&iN^g=l$Z2uy%Qlt=1!B1n9YSZ@o zU_7S-0zEe>WCb)8T1|=l2UbAz}uLe1J168 zdJP`WC6+JnoaHSSJFVRo8sMj06--|g80b*tcp=O?6VV{jph2Xu9^5+R|UlPOw8}&CB`RwM4 zu|7l#XZljMq#5-P6Z|g^=ZM6Th0Ay%1FMl4wA_R&+mO4Owi?z z0lDq1(G#R;tqV1{qoXQ~6aB(dRMSplN7RoXpo{4??{qwAe&}M8BgJJ-@TBgsR19jL zPY;92v=mYM3tb3O9MF-llS7p7w8jC+OG5-XVZf?i?5?j%e$&XwsYd@wlVqpIe}~ag zjjO8m=Q+7xo+$sh(k*a&9P%0|iLFmb{wLiGOTw}+sZ3TrV|b?7G=%Jp?J)!*-YIsw zkrv<}d&g*%Vwv)liQ(*hA}65D+QsT%I&;+NwG_~~Nn`4f#a z*L9G5-T8KaJMg0YAion)UGnO3Psn{eBIq*8<#AZg6cD`OC2_ZZwwAae^nm>W*9rRf zP1e)it|I@wA}Da}!0a|nLh3%RG2e6i-`Qj1T+l&QlsbuI=la4nsaiVLD1jv#h!~Ua1;O# zU-7*6A`b68NOVBl>Lr6=_YT_BII##t*1jMB_OQk;;#?;yDi~l_Zk9DflzILpsH^VV z^PRE5t)lR9C%*nk=P6pYP{yp>5(h6;(bbN29h1@{GSR@vP61YyPA zz1q&vw_L`(&UFE|NAKe{{dXn?j!ixbhzg3bM8!bfb6p|Vb0DL?m2+Ej^crLy`uZWp z?cZoxqUbgx;C8i697d0>`?0TqMGr);lq%;^>S!$oV6kupjmwzOA-%}%Z zn2DsV|~kjgkzQFL_V&~;uHaYTh@B1fjIvD&-h zX%&lTFqjiFV&mvi>!u;a;QhpVPakm=IxsN1wbqP zAYOrC${mnOCeL3lT*Sdp?4AIWg^Y6$L?>lUU`>qAOCj-2HFhmKdag)}E%`7C@S%QF zB4kuPEh{LI!NbBLntl^y5XU?4CsjVH%LAXcy;}xR&^dDa1Fs}zj!c#%EV_tZ1k1+I z)6dejw%4h1d>q_cK@VUl%pwb>AOqox^2=6tjKAmS)Fb}+9?LNqrVDtTIoyv&+Apk( zt4AopPa{@~r~pQEv0nVgZv1ZfOP!iS@CPBXhNuUv%M>VGgl{`8^kJU%oC3Zw+2 z3O=rjnmv1+-A!q9-Q0Nv?WfpDSfg=A5(n@!uAMO57fg3QW&c&Iu4fjz`T;|cXg%Dx ziSLy>3p?m~DR5z(A68v0Dgi>YI^g-#@0sF0y zY99J~fpPc;1(NJl5k5&i$sV`mNLid^aNWO?`pI;^)BJ*EpUoEd!&827D1#>!km<+o z2|#qt$>b29O) zB^4jtrM)7(UPmUGi~~mRCTmKxA(87wytW@{=$E!))K3QUJOn5NQt|*qgdtl+m}-_~ z*t-asP5zt?`2zVV!}%RRb*lYn?Tte83<*u%qwdJt>#I0r?NtsUrL9_|ax!v4VyTaA z6Sj?f0{dMW$psA@vBgTUbRhchkG-L~zvU5yukB*DN}qsiF%YPF#j?Xa2lJ5fWlBfy zgo10o-ZBhQYm2V*TCOT{%?ik?!lL~?In>bM``hNyq1Srx9VGJjqlF+~<{(+Dr3*aa zwEK1-%a8*zHUvL&gfAyL!|3mQ)B0w-cb>bm6BR#S$NOPKv8Hl8Mg~q?!%g(uIIs^fT3b~5t>~NuR7)D|HIOE_*4D=ZyzHLp>wQ`Q8>1gaqKj=g2?d2Eu%%#4GCP*&MHdvl-9_jmsX=kYkN_j6p&YuGES{&bwnoDxvj z4|#k=>m2>6_0q=wysuhz?~FxKkVST(@%mpYr-Xg=YWrp#o5kgEw4r+|k}$GhtG0(`JD=-EF|UN0IY@I>S6HFJ zE|3TaJl?_^<#VNQvD0WxBBJvscJpT?OU9a@S#y%~bb%nnYwcS`+`;rDNz9+cLq@DN zN+h7#f2q*A)H*;O0f_;av3RlO1quiinMlbf_P4@nXVxW#W5c=BY-3)y;s|D`-Gcgv ze9a`t0BS9@vxz;zL4(rE6LQVE|;8Zk^2v8IbLt0drZBMcF;Fy$X$4tDT^x)OnhK(0`w{%|PEgf|C! zZ@yln2pL&aPfb*h)$bUW3%40*t@p$T1Ahi%CMn7QrNZKGPRYOKCaA#l0jX4kNcs40 zf*4yAKu<*My?&_fvSoRDI_7fY^J0vwuFQJje{9tU#uc9zf^KIM!ip8+9v@v?HMT52 zWIOABoayR+z3T6`l4h>}a{URUdvcNJsdxb^F_+lOa8y{mx;gN?lLN{6Y-lqy%6mQ0 z)5m)HmpA*}9;;HzhWu{JVZ+V8-&%Kvx6H_MTt@HsZ;I!f&gY!YOjez(e+snxbfWx3 z>iCw>P3zP9p8vZ4%Fz4%tDlMHIXl3fn0t9~f0tf6>l3`;_Mv(rxc&SxFV>R5(?a~H zn{M9Y_{-dBi?7*{Z2kX6bhOWo>xZjOwiZ@^c__KUi@*E+zYMCs*^t|4U2;6h60LvP z?$T2uIb{*rI)7dchF*DmQU-NG>zdNM3XFMpn zY91x5=av@4%V-$gDK6S4wZ?EV=Q@&Lr?szUBb6z{Kyl9ssBgrMI9@T;1q=y}#K|S7 zYC6{;1m$Jb;G|^#C`lLMzJFwg!}Q}IydQ+NQ{J;*O;2x|y=m>bYdU?~>r1Q4eCvlQ zbN%6?Pjjxj(URs*ie*>qo2~o1O}~T$V}&iK20(-rYrXD@5FilOcQ0FPlj_`N= zYjTBK8Gwa=oU1X!#B@xh#zT7DcswC#85kgtU>7ZTfC~*bk2gs{Fd!;49pJ&k+%~#D zRMWWixa8~eaO{q_Rn{R#fh>^IQJl&Ir`AkO)3cv&Gl>9ySVkDAx=dF9tN+V>|%=Bj0G23fwwvz)xV zxyyTXM#SrX=y-Fzpm_QVWVJG{c-*ip^!6;`V!pEWgpft#I`bmZFVMfzbmviGi=tcK zokva57UrHawVVDgq>+C8n+k7x`y|_Td25!w+qa*3>b~24Y3YaGXz{cEA3Dex!KFjx%apTQTCXpGV&uzB@Sh?J;+JwYj=G z`$FHY@*G&RTUB}t=7}E&E>jn{8V;t&zgQHBmEGUl zJf8DjP8To#Sz%HCrg9J_tT#6P=$@*PNHV;M4=mG%4uxa*13t^}TJre~9FWx)ED5Ws zPyD)l>xo8RzH!J3dwzE*4!)JC%2n?sX$Xd^tiCFY=g8*@)+d1<&V}W#4UL4=Rjj3L z$Oi?aLWnEfU`lZ9%u{A0f)pk`aqEP^xcKvcZn8{Gs)goTjjyawm5%NUrJ~K+knD<$ ziFH<)UR&z=2zL#gKf}^+dN7IADk0q?<&lMqgv8vs8xIf9uWb+emNAT^S(b5qn(S_Q z094oSfocH#?d{gdczDSVR-#g)<;`LqC{941V;f4MeQSR$^I%xI0#_8rP!^RZhpR_RPR`coG6NhJtbE1<%}xo3 zO8T`LPkYzsfg9$Ma;nYuV(|HKUE&jkv!@Sv{7=5jw>mHOW;nK;Q9Hiv`tE&?;9A8F z(10r&-SPYPp1gCVF!rWZLjI_INm@W)DOgh8lL`PJQ@o7Lq4JnS`n|%t5Vf*AYI>A) zTl;2qO;qTI23oFC{;itSfE?f&bo zpKZE#0CN?EsJ!}&E{HF%8F1y-%vp{;p0W;O^XgD0Z|v)}$Zn9iN9vT#t)_Pi23666 zMJl5}3?gE?nWk_RaHZ}z7&+mkJ9CTfLa@A>CD;mtR~x4mB^*(>G_)DARJ(y&Ev^dK z()2V;6cFh_!z{%`A`FcdaTbF^2*^U}?l6f7kQ++buU;914(*PM=*y#&Tj`>!eo}-zMOb1EL?iR$0M%;_0^Q(1`(;kTPqG(Qq=CelFwt z!Z>C_JAxmzenN8?HepVn7eXGAcN?MKogY<{Waqd^vvsFd(ruQAit<50I$-VVM??tg z`~0+g=2G_DaOMCkA0gd5H<*?RKMv@`i#*V$^sah74!|arvP0F%+w(AT?3e$-6y+}e zbv3@4{*U@-{`OA@IUyZ_#B2ONBSL}(z{0tQq1?E#`M=TPq-9k;N_A`NCRqGmPXY- z27ih(KfHff_&i1azKzxl2TO_e(cs-nL#i*GdwmhD#%%60#rBsww|P5)2MU!{OoE zR%mg69zYmLL_hYCP(#I@#T^=jLe)EA8q*|k_e%;E{y_C`O*1@q#p)B-EYdtHy4fsF<)c=_dPFd#DE*cb zGTnfl3@oF1!*CiZ_hYgRbVf6mm=JAxz5xkurjivmZeHQ6OLZ8-7>v??BMU zPv-Kpr~Qu=>a4p$XWjl;UTr)b3wizfQ|qS1?z@ZM^VfjLZ;AwDIot1ioq)X|gx-yM zt!ABcM11q74o(|#UH_5bo3dTo94(JYahdmg^yQ(&%)c34p{pm&g`bapv-$Vk_`i`l zsJs3AW8Lsu?+h zB2Gn4^1QQ7?=Y@2Wu^fMp#h)Xb@JvC)1N*ovT>iL+@VU)OMPwx|j zl!-PiaVSN}`I=##= zFB@w>JQJ~il6H>S3r#dXeOdl+9JiW_U~%&9AReQ#YTbK%v3m!=5llGIz4pu*WO!=s zd^q}8G0ywVeB0Hy;ghScciXo=Rbju7Gwrd_{nnBwVS`1x4jfBO$3(aNfK zhNNRwTAEp%E$WIRYc=OnBYetv*9on~q~dGTWFG{~{+GF|P{JZ~71)h<*w1V!>NQ?3 zA%C?zBD+8l_R!qRaprveOi1qQD;fa3v5%|Mgh+H>7YZfEMsSXVO2Noe>L0Yk3Iy;# zMG_GDKUgUQ96e|)lTs6y`_o~G5bzt%0c%Ldg(_!q8uF7}wWXZwO#Qe=IsvC_N!ttrFCPjVu5NCZ6rux6w24! z%8$SnvQwgD5?JF86h>S;w1e}alIaGh!c&_jFya}6g>WpHgA~9QT6CZdFwMSm?weyb zZ10urwUT6-cv>J28M=r_j(d7G)vi+hWiiEn-FG=` zSts`LE!w@!kq}9h=J+JW(|!e`LJnXjwIVULQ`%)JNbAO`)98OUdKh8D75BXQi|w}! zh6hh~GMhZn7j9w?Q~{`1zn1BZI9E6zl(JZcl_sDA#1Lh0>bQSmB(GISY6o1*(v=+{ zi8myK3Lm&0=y(_=z8-E7IUcaS8xcsV;-XC{rl26wWc$L~r*2{=3A$A;=UANjm%!~6 z?Zd`&P|15wq~=?zB(aOAi7WAhIbhT4qLI;{7FT>!Fo5S92O?&g*RNB$1X@B$2roOzMnQ>MQerB)qDLv$#1zl*E{n(m_*?tU}IM0E0o2)uaO;AP{jp zk<>0BEqcK5mH;uJ%$OBS*l&1vc=zR=Q156wc^?4%E5}I;mUy<^GR=YVpqb_mDC5?p zE~oTx9WJ8EN|J14$2ZN7JxeA8wwk7(8BqY8^?qZTFaxSe!3$ZB4WE$^Ae40?!dg&E zI1&hF*Aq|k+BLM7BNokz{sc@;8#3@Tv5QevAIJ#V%7e#)^Evo}y19&h6s~GKZ%7~U zBCSF7MD+D*?-DFw2sxbX+pirjkP+Y5Q5YQ{b|Myx48HS-jJpS%` zA_r0%_SII-?FbXj-ck8vg~epnZXYUDCPTD1TyBtpVsu<;)uGn%Fc8c{)L}$KEgg~! z5lvSCFvQu>Y2Aa$C}w8s7k$djc_uT8apr_228OovSOe9U>b2>OD@>9bpV`nYQg&=H zd#P%3V@f74Vxqec9pxUdkl2$V+H#1EQ6;9GVw_&AOrEdjhBiK(jABa;BDHH6mUN=f z3n&=79yl*l+Y;-kEqo0^e)}wdyrfi$O7OY^{1(m71EK<;q9{Q&RLV*sj6L|__2-`V z4gOK*VtVjg`r0XWKrRtt06TLk&Uo)jnHCENe^`QzgOU3!I9I-2Q`tB^?t6z3DVBCL z@!eQBACAL@_O92iPl~wNLlkiU+>aa0y*)5&VvoTP))5;n6)UBWmJ(RJ7L;H7&Z&}&U7yQ=Pg8FXo;N?)sn=|g zvALgHnExFV{++9sS-jjt$Oxckw`)UBYO6#-j|>N!5AX(A$Q$d~>BmCqFl@UR4@*8z z@=~`akCLPK$LE{G??cTANfOoXCd&U4?^Cl#_^iNkylRlf?a;rUcK!A%B(L|H=64Al z`qxvx^PG7v{C(b$X_$T4Z$e1VPdDdzn>)WQ*T-~yHXpaO7Bo9=QSi23kocdSmsGkP znU(*qKQj%8n){udJgt7-=CQ4bg7zXW#B*E+njd-|+j|E8_Ls608oeJ{#$amyb^EjZ zrhMbIn{!ND+SBbX9%GR?^;QZ~JY$nIj+fuJZqA&&HVYJ5TJOkP?HzW?pFcJhc_#cX zh6P3R(3M`FGk}c&tWg#%Af68YS7PU&fdd8^k!igb9Pfd20^$FmAr9D9OtnKMMo zii=d#N=d8$H_L|@`QbD*$ZVSJCAzgU!{jk6;p2^)79se!kF)~G_`zPntWa=$#xqfI z3F*--w*cX!+EE8~lptk}Wctty#|tXW+a?z*pLt-N#96U;?^_&oGQQKrYnIloptMoT>5|&%XpZxY+63?%JW$z zBlD71ww-ATugYT`9pw94kF~o+y6I07r7F62h3m1U>JU5ONIvU*owQz)!;4y0iSP^A z;yl4T$%X6hAHX(IdBJ$@+JyKt0Su0jT)k-$jkQgX;=2V!g@Rb~N_oU%aJ2D88vP6z ziN<0ZRrnS9ZHx5=HSfTU-BRXu3HXxyuGh$a>FJQgP0O2eg2i*m>>I@`maNwGx=4lV zy!NYqJ&j1@ts&m*J7*Riuxr1y8Gvi4i-wl?zi|cyJ zgF^0xKHUEv9(VKXbz__R(e*`jP9`fG%S#fapGy8N2@NP@X(mEbFN^ zkK~|digcJ$9$aL1=u-YuW=8LojWFagh-QRx+6KH=Zk*hWexm(J#%6Y>qK`*;^o5vQ zy(1V<9^8S)r4J9p$4PyY>rNn| z1ji$hHtM6o-COnei;Bt3qV7!4*GDhZxKBIaUWL4#b1V*j-_d3iV)*+_Ccf(-+EP=f*VpO$$;3CXp2omU%!clRAUxSQB{ zu0ug|FO`!Hg{(9@^W#M0m_&+B*-;y?NK61o1@KpJa9Wx{XDmh}6a>FTL6plE>YxXf z!NdEtO;Z~Bczs}wAAct%Cc=T}A+^NRtid?*zb{p=F-!(49|gdsMMWAfRKh#s>-cPd zaiqd%eWPwJO;dX7@4F+zG%87NVh6CTfh;LpD!_ymlVUPTbSfz-nU#W8gdEMGrmFNc zB1AbjX-+2@%3l@+P>Vh=nRr@;LmJ%sxKypuVfzp+tkj$SqipY`bzZ8^@<8%kb0-3l z!gk}d&tfK|xZ7pfvh#_>&uN)q{)?~2B=yT2*Po9SJ$~LkndYpJj*T`|xk;(^b3_Mz z)_gVmYjJMB?c`?UDXWCB|D%cjxFmlO!2e+N_QM~8la*#ir+sV*G?imIgD05r&H6BF zpK!%X$&{q7zkTz~IKhFQl%F*&yW$zoFDvf0{DGrXx~q<>`QJ$SAG5WKUtXgibI2~X z)|#V?NK6l-^6RFqQV`PYJ_{MG#sOXhWT@&I0M{K{^I4f5I$2#^L6@Xdtwp(`?L|q2 z0&IpD>52*=#A3lvdVm^PX#i#S8V-h0;ZmZJAPQU)&4(PeQbo|*g>mlFwB&Xx@A>dc+YNX;A&En!9wJ8u+40P*nRVlsXz=`0tR z;Yw2+M^LyxqAEMoZJ1g$UK5vUk+Q78l5omgkL7=?_RuaB3Qg3Kx##j|lq)z1566}Y z1iwOK0dS?%^HPJaF~qz-@*z~{63?yAj`g9+R7Gh)9JC~*X%eh=RNSIxKvdL$3}U7| zYBM^}durNrO6P}<^A9}-jP6?~{-9*S6460Y0qj9R{OlZjkO-hv_Hx#+N&V<|BOQh! zz6b~>UA)aMggPQ$xE{ipDxMyVE>H86*)HSX^(`NJ)7S?O31^ND#nB=asUy{?;^T~{ z^}_i~=dHLNXugWUCL!Wi7!aL$Dp7V!k$c@eTyH5ln+e`Y1x9c?OVZjv`_&j;0|2@F_oU-<0meNKYdhD-NC3>+ zSOV3K6u)Jdb3UFicUWws?MG1NJhE)Nw7w=zHTl(gO2YO5J9wGWx$?HwIkc>|D!Y6?Ab^ORyD1J4nF$gbF{&L|A^`_ zAb8X{k91o<$uO^LcHZ4q{BrhX@*ys;L8dbKqQd(ut>nHBe$jACt>(07&i`_GICdE0 z8BgUBHVns#=N-5zk&-Yo7OB2kO49_U9q8~C9TI|qyFr{H_JsyXFxleC>Vm!S;h1e| z<32U&4|YpnU|v483ZJxyB&N(Mqh-v9oVpY`ZPMm8R5z#HxOT)H!I5whOp17chYI9x zNkiwpyq5Kz&t|o3MoEK~qScJJm58%8hdHgXy~9U4u-HtkXSAg3`&+E)5%-P7gbi2a za&Zwa_MZ|!Y$F6BjskS=3;nbf98W(G=YSPZB?ks1Jf;8}+SaBOPJO^#6Ktk}00GmR z>Wm5Zp@3qwP}-ucqj&YPC5>~uc~8($Mh)2+Jk%ie1wIKWO?Y`wkONI~@|>eECWmg(N5KlC%P;sc`^4uDSBjS20ht0VvAqaFJe+Q3w#W%i{G2DiTp9 zvkITEH+|{TA}=j1A<;YBAp82mvhKvHG+i$)`VNusQ5LLTm@Ym&ywF$wTNEk^mp-nf z{VKqY>6?-YQ^X%DzqZt@_1)CbH+RE|5D0LhfeyA`kGjLCpDJ7}GEhBO04)rSjha@b zCnXlUH__r6(Qw~uS-8lyB*3;3)73$x#=Xm66AH9~PfVzY=NT?`TIHp-%)ZI-zdkF; zbQ$1u;rhU~-+U)~Z!dcHf}&Dt-%IFf_UTP0zmV78efdXr%e5kdMI+aij&cisPp(M! zRiB?#-N|-6cXmH?{tu+y(w+MB`uvAhySTXcsKXMWdPXrkNFnY>`F(LTu%%%Ymhrr0 zV0`EF&yeih#CgngI>Y1ZmENZ9~d$qHP^uWZnbN0cmC)y*GIXMtE$41$!^)rcj^26@nq`ztdCcO}f@Etb4eLjPqzmd&djK|yQlUB-$JFotHc=e}7Ub@!ytPcGp zM7wP1x3uEgi47kwZ%L+WF|Fpex+ox#GMrTwW;hma*~g^@L#TuoVb$r;XtW_60|<#E zv+~%I2lNugEVvltSi5jT_A!r(l;}E^$e59bvyf|*$(aZkfS=p(oV!E120QV(9$;%$Eb{~f2GWmfRV8)oLn=wjv$M$SKbDWK&KV@n z6K}RN{ExfIIkBIY60~#`&ImzGpI*&;-urr6XvY8Mimi#lE#`V`uu8tjx1Zxdo%~x# zc^>q%^!e-6w%@adEcTby_k7L&yUcxIGdi+Y{Blx#b555bA#c6dTR>P5F=X@Cm%jf! zJ(D^C*#!+c=>Dw#)j9MGoC)Z2z^QT57(THg~CtX1DV&AygnzpM~1KSRf(ahrr_rkK_9mTZ;2Fuz?E;*K>_?<_?YM{F_Wz*1iU4lVaA@ z+|g(UGBSa$!$k-#%{eJOfC?29`TYl<8aZytW?Uep-=G<+-uxkrh=T({iy{Sa!iY#J z8DwIa1XhQIu7=Vay+;%;9rAqs3O!N0L0Wo`g<6J)uAN*s`fi#jUjDfQJK!|c_Pg2L z8*Y1izpih$vaeh4#X4+$@54yNeENUP$yydwc;lO_<2fQF$KuQB=+<{S$6RuI(u4J6 zBQuRh$1fetxm43lJ}uq3S=rtstg6Z^&oN%ruP!39*e>S&$q<>odU^W(Z>W7^_H#O! zlhvitKLm9RQ4#%WYosP4tZ&vJNK-Ai8yv(WzW>TkI5lnV8P#NzXYV_&0*%bX1GtqM zyt6Pa?vS8s7D4{wWlufpI0gB^G*$VPo682KB_0j-BAfr-$9N%#@L}N$f;ZFZ2Ph$Y9%S17Yx}#puIt*9Ex0Q$^ib z^fflc{D%PdGvSFJ!x4l&X9ODwYyp}4~p$~9(32K(P06d-X+B$-P3cI_fBSR&24*kTt6CNbzPZ48O2+h_ClUk znta2n^9kU)@xdm^v`m1~vLw6d-4hjkhp0^Nu_W1w;6flk0E+o5$HYhbZk1IQ%m7|B z*Syq;$ux>51>kavF%AK8n0j5J2W0FVJdBU?L5V!}@yhgeC_@5=)uw%ZzRC9;y3cKN z7naUxu*XmH8A5Zz=IwaDc-l1CyZVpU@xM0llbI|xwFH1hfE6)^60s;bmWncg1e=?) z%AR;X=72&#K#CPEH$Way8i17okt&C(ssl*_0UU9GR9S_x@<>nW-OnfWKV|$Yu8d{0 zY1@1bowhIj{>9g%Q5rr^X)A6AXKSvY9%AkHzQ@1i2_Dg#BKt5MV-p5$s!&pCge; zc1qY_I9#1$Fe0ByWH2%)b%LrXkDZePH$!!m^ z@^!77w)QL8KedH)Bk6M<&l;Jd6;C>!ti{(B?DyNtU6(1IT&6w-ls-Ii|3|Lnzqj{i z;v%-cIQgXp!CS(nf4_p6&2Mo1AB#+odMQ1&4;T9FvlVNJKuIb>U9~h?lpdvx4C=5; zLw5aXYrT;97zlTzp~42BGGo`n5hVtyuu`MHIjHPXp}9+ssq2c2h@-c1KZf5VXzleib$9^9Rkiy zjBv`Y!xC}SmJtO;00LHJXMB_$7`uH127w484ILdgtMH7O>3U@qt;#~bNCFp{_QgLD z_)>AOPJ-hS+cS6Cg7CJhrMcw0K~63O4m3MiPHsRD3-qAS=@x)&sl#f1j)xuweKdeV zqn{x=wRP}La`z*7#5Ubzta#Fe^-}Br5EPszN@TG0MY1zcJs2naf`XubYn;Jc7}M9zg;jDPF^QdWvjkM~G*dJPxn7o~mA zEiMKCIB)h}rW0_eW;L-Q9(mH>p<%{+2|+tzuxF3a1hA9pTbF5@8X0A=10OS_M{)|O z({SbN{;K>YWZr9Mq$vY-^7-bh&GY6LUE|DS!NJ#!v&V#U@jqD38J-}Xo8OD^H22x- z*)ID;cvDCMvxZ=t?Rx+8utbpt@aAfNVCQ$}b|5PZ`jy;5ShD`nmqixIS(oiUyJpLF z*E2VJ^>@5xI3?fA69c|2ZT$J*x4U5GIWQM^Z{6~3LAfrq5)sDdF95i?(ypWyXA%0eFg-l2Tv? zN4H>^j-&a9A-CH8Pl^j4tNqXG4-lr6$Ak<0pi$BMCgH9WXx#h$x99eBts{XlnbW-8 zy^XXeFcoJ;F79oVDvB%0%EBaJFcurK)VR+HOGm6~5)FG!4r#?n8_oBr$lv=ZhIV*$ zll_GTZ3HuuXqBx5ltPGCz49>bv`{fs_;Dq-IEk%_gdTtsMFN`W5ELk#Iu#{aY^(3qL%`|n|UV1Vmt+cb4>zMhTh*=3=x4!;D<8Lw_>V9nK;;6b2 z+P*k4pY^HDt)YXu-`l;ZcBZ`?D_+?j?DReMb;I61EVNn+5sLjIEJ>mdaJTp}flzqJ zMo(mzoB&K!M@A>UrQu0GO8G!Wey8IHWwaVKi8?{xGsGl0x1z!WASD62cgLUhkxc~9 z=%#Q52C;z&9LmSmb*X-LcBbOgCpJ;p!~6f98a9(Cn!}KGJbH(HojfbgF6cH*jGK=f zEe}FBsQ@yu-ebu(=Nquy2P9`%yFRNAp6^9GulD@L{h{S*+wtbcH?`%z_eCgP?uToV zev7MKcDg0 zzh9W(ph1{YguZv%M6f8gg^zW=OZFWRU_L+;VPOa zlQ4C%e!RQ9W|@a>6|ZsnLL?E4C@6wQT~}EA;V9ul%Tk!E z_T2x9)w2c}8Mj|Ac(Z)=_pnp4Qf=3`_k?n#^{b{;YBAomp!47D8;_nptqHE#Z#!Kr zFX}2wvYpm$-;|w<&eq+ON`1?V-bwL&Jbc9Jr^>sGVE8IVLdI;EW+0E_8pN{Ok$QzP zINVC-r(;hK;s^ub(hVX^*PCuBsat8TxGtoKLCEs!_*A1FbY0`eee_czq@y-i+TUeG z^}e4YQ~|cN%8Cj>k-r#!Va*qIa;%&>m6lWI0dys+_D_wUmXE4$h*jaH9!8XvweV3n z%T?_|*(Q_Eu0C;)9A5Y$O1Z)M$cZbz>Z;8a!8(XGTr6!wM#qN<{6e?`F0ukmPvDCq za>hc-svZNff23w63y`b8(7LEcmi@=JVf@0J#Q7v{;hAmktyodCMd1l zVj1RWAQKo{gTov}J9EWGfZkc+!>Z&`ZGr>fDcuz>>^RujYet``bZfz{GgZ2(_i89O zn2FgrE_x-wN{xJjsUrHcx0KM#(S3v_hal7-ejN(xx-}aIKD4R6_6R=3(9l#L1S?H5 z{r7||Fiq)UQ82cf3f;>J-o&|xJ+YWH4Ye|ODo$cbtMfvPnZ$%SwqlZvO%C{;Lt|fF zh!iHOg~MsD<)n&8cA4f!*^;4*-{;N2U{yGvmB7$qh&onxjLcpl0|CFG0xMPKabzgl ziq4x#T6#CGa3Ls|z)0%;@g!`G-5Tp)!k6mP4r&0PXVG$RCebfwta}=Q_J$|2keHa-5B@ZyGvM`;IesgCj6-=CYG<6H`c`v>C(XPA`q`D9`(e&@^ zF5AY+md3>mW|>WK>Sk%wC3QGi=UHRu@Ddy)LNjWs*R5Cl&Nf$^&&KM}xIv(5gTAH( z5UYf8PtaS*0#QQMh{+p3<2)T!7~6%$?kFRIpm_p`ll3Z9TLd|*4mJ8NZ@ernk>xCV zTK;FhCa1$VtRVT1RRc3MUgFhk^I74xD+?o)v%6ggmGMPu%kQl8j73Nyb}~zC;xmCN z{!O|xFP%Le%B|7$$4QKtI!u*IFv5JIBT6ntWLic=7=|E*M~U!8xF^Gtm81ai;ZX+6 zl1+DC<+Eak9f{`E$pOTNX1)Z2R;$GttHdLvS8Ca%rpYQwCfg^tW?C=_3w0QvHxPv1 z>-hVWDwhmSucgS#)aFIHii{Ca}U}98q3Kj)I2&o&` zW!8cGerYs^Yh5%p!YJ^4nN!r5p&?oVLIp4Euh%WH2?*_`ugfn4)wNjBkcAdoVSw6j zQmb5Q)i8O69@g;70e)_M5}1~|CQ^k4&4ZNRInsfLk*pn6NCMqeU~Y%?FVdBQrm(9s;D030!_K*NkCsZjCH+ln3lK*#X`o7m%%x_2p67Yml{7yD0qcFrvx@Xx>fB^~QU zNbp;cUVU>Z(K4L;^!b@88;|4v%}~$ot*2*Xc;Ri*TL;VM+GUZi=Qm75-Tqx_Nu+gt zxW3Y~1Stsqa++D`Yh7Tv^Y%3DrO?^B-< zPT&_ZrXqkt5IPeKEn@zNZOJtH-kn6<*UZ;roV`R|D>BH0^C$4Z(Ocw3QcsBcrsSL7KhrX3Ot-3!)vf}u zjI%rAm*!hZWmRLJ(Ue`I5D1)>&*{+Hg+yhy-{^g2maepe&msDfR+t`%x^XI9VA>YY}ov8nXsRViB- zoS0NeR0oiYhD&+PqLp6_t&@<7SNrK4ZZykzOIyweR=0_?Re1tWYzxkIPcD7yt&{RcB?C^Qp0t=n|SMN zXZJ!(Q^aj`b<*)y(p|f#JnY&2bq~gzA#1p|(yZj}3bpW}x>~4|t~+uPnvYp*))YwA zdhYGA1^yL!j=g7?BV#$IVO`&Ng{#3{AZJkd(3`n2~x~FMM>}u zEi`s+D&0q?<|Ge9un6+O{#Mk}v}#KY)}~ikbJL{R+Ac}$TZw=hiS@4CSEdWJ=J|`R z%mjiH*MALU`G4nm|2`K;{g??r#gO3s{tUL9}TloRQ!8?gN_{!uy`!N zi~`CI5)qCz6u%$sWMKjmah#Ul#q&4A={u(#LKbi52Iu_kF;5!)|LlDvpG-)z z<(eMOAn$r^_D4}otWm^!t~%3a&wuv5>o2c^?Si=lCyQ_R=B7oqw*m$Sp1m1;jW-h; ziua@6&;Y1jIRI+={lAmj&CZ0n_r>IKw(3*YMofza&~`gON_E*AM3i|AbWNN}M==1> z2tXW0qW^7znzE6Y4QQ-^#g2v^T7B&8)yS#c;2_SrJ>xZrVYWMf8E!Hry}cRu5mE34 zgerl_HpbaYmkuS(7Rh^)w?2QKv?=*Id;YVthqd+9*Yl?7>AK(H2u5faXg^<2hPw|` z2RevCt+~l$>6Y+Cr1pTMM(UcYkUiSILaeM|uTVU{Ua8j#q!NkH(TNbNSRmakF8rEs zWvOZ5SdP+X@?MI71t&}Ub(u$Z0op@(4{?;CsFiIlPuo=K5m!SxBo{C+EiKN--5go= zvzJ*I$nmyykm_D3o&yvgRFveSOA(G`s^JgT`P4Cb+nDCx9uK67ThHforl!l8C#Bx2 zB+6?c)l=384(yJ%VI%?&Mdowm@4YpjY`_sIwa+^qPa#$!XvMfy2$`pro)@J2+!`2S zD=Tf_R7Z^V1hcvBSPn5j#m%Ep)@`RfHCC`M*VPV#E7EFTmh z8^gi&EtkQ4Hy>V`>8_v4cUvvjT0OE33Z8B2 zHQO1!??a+jwN2-DjV=M?w+NpQ_3?mcm{I0Q?A6v>CnFJV7mx3KzgLy@M$2s}yZ;hay7^0+YP=)$8x_{Xw-3d&a@aOLA3VS&~UP0{yNe@^V>(J6qkYyI{Eu+HN6@lT!*oAHc_VUf3km-8=xdWGhLW-Ku)>sR!Kn~x^?p3u77NWUUa7HucAFX;6ssjILQVnh1ddY2p#8LR2c3By z*hjueTZU_AvDz#^I1x0pd|H72eI2F~N9m5prv(sV=uaq4j{q5OpP!w2K3Rgd6Hfro(N@o5)yi;!8bECRCicI||U@#>+E}sWSOi4-^O3kS; z^EF?CiW~$a;x$<-c1G(=Y7V`nopUxDc#nMkgtH4$!1k*U5e8IRTVk~#B$Wo-*D=%& zCiE`>@wNjSN1wTAUb&TI|D&R-#@6j*$MXTt%L{_c>R{ww`z`aE6Sg+rqnG~2^Ea5g zfM9|^HIGH{W@rBM;aBoEvo-65t~qBBidUzT3RlgKlL-Wn$*W<49Y=lnHdEvOhqKzs zbrr7)%gSPy)G^dqHgJ16mz|bK$t7-*fI_#f2}|aqJ7y~O%N_fxk8Zl{g>IJi+eiT4 zdv?6erny%huTNi_t-|Vm|GWV>n&`UUckAquo^zxM=48)H#4v?3Hlecy)FCQjlw|@dV&B^idXHpS!jQNUhHP8=Wd&O-|NwjdcbAr6#obz*K+)pv7Oc2C6o=w|vj4Ncots?dBI}uJ%{j(= zM+`;DU^H2Llv;6CHliP_Hh9G$xnnP*LvJ;RIpqL+Tfu!UXI@VffZ@x69 zv?1~HN+A&vNox?idKJWg$t2RHP$ zw=L@0f{pH`>YHg~bu8z~l>8B3EhuMzXxsJeb0RU~zi~MO|&s6>L!vJ!nFTC>QLZa0zQ^IrW#b?xbRj1Bs z<$vc9_iDp6w-ezx@s`!pBdOxe9b$3$M%=~W;sGlJ#xBd3QWtuU$$dMA;!iZIUV?@P zR3D%J4OA|kUpXRoF7#Qay5%~353OfXG%mOkzy|)G5k(ZslP0LU zMr4;Zh^(j_1DS&jEyb(> zaq`eK!aND+Wg8hr+^i!tQ$taCf0MW`T9tJXwr$GVj~>DZow}b=)b&FccP|i1iWv-~ zbfS>rk~EP1o*rCU@Vh?T(&pWt%ghX>v7daz^dK_}yP!^(%?T%M39O_jDQn2nQ)*dOA)see;6rByVEC!ZZkcF-xe2wUR&rfpg7AuV{lDw1K+F;tc?FdK zDPrMLz*tC?c(AQbk+O6~b91LW;P~|OL%|gFS^3)ix;+*^L-dyUWqnJe;stRK1I*5; z)$&WF#!f?ZeV-48UDp{E4R~YzDrR8$NaV499nnAC>+J$9d{YWP4;@H z>n{A{^3i)lxI!JGdw<3DEZfGBygC#15qf zG7?w=w+cu=Yoi2+V|a7_gs^ATao*I!V7mM`rZzs!i19qBsZ<^&onUD{_9QJ=*C#71 z%~6>I{X{RDiSEDqq!vXQ+tmgTS8d)*0Tp{gjFr6iuNY{t^qDY}=Ka3^ji9Ebp+$G& z{mLZ0WoxdJNSG{?K8U{FHIce;n%sa4&{k)O$2C$FEg53v+x-sn`V)?)npw^ErDTR- zQR?VB)===v?a0e@8o{_v7fpZ}2><|p9#nogIuL1p%z1gvUuxI8PKxc=<{wXc1qkxF z!3T8i@~UUtbF2+n_Yef!)!|+gpWg1>7_SKn|JQ<^7V1DSL_mZInM;ICu;=LL}P zJRi63xU5096Kps1{Tb|kp07uRnQ8?Sacs4JG4j=MM;UjFU5d=Fb?2`?(C)D<+Os;H zG9X&n%wiV5e{M_Yar_HcQ?*pdH!Y+qYnhVzAStcJ9fzf#2i37i<3(k%8sE;s(+G`~ z)u5`=>BXl4wl96p9R{p*fBjl} z++Iam=?sqHg_O&7~e5-3f+WVe7@e(Lj-6C z#gMfB%br^e7&U&JyAXKXbN{q>nPlvLXC2;Z;^cI;qS|^ijmVw&`MiCs=WkycG$m`wdiM;sqv~ zX@;*NUglr|9(Tp*YD3uptF>0GPRyIeV_dFlpL?vL0&oC!ySs(PpW0aS$TcG zWy;8pX0fTJ`Qt;1EW^C_T(-#Rd>6*b!ovJMEp6$Hdkj>oxU-8FXw%PK0#g?iD)<)h z6tJ94*r4o3nJG$<(7Pp&C}x{WM-u2S!+?xO7AuDaOVCm6mFb;}f~zFl)3K_U5U#Pz&kX=>jTA;MB>J!tv?7c3AG9{&fwZ09JPL;4P+KG3*mxvB z0RucZo5FtGm^ra5lW>-4mS4?|D$qbv7Ip3HpCB%B8kXkP5&w~GE%8s+0^7$c3BO@; zY2K_J9%Qwa3lyG5@u-7?Hi!16*S#+%Wq!tBS^_7Ww?1``WQfQ?`!_ zEq_i@|9j?tqEgN<%C(os_x&dg;qV$O8Zd6&>E$g(GAYStz0XAt(g7pLuk* z{$j;3WSNhDIR3{zNLGnLW>W2LF?3@Sj$!lR{7hvOsj43F3= zr|BgNB#Dqf`s{5x$p{S3j|Q#cMHO~y3&eDZlw`JYsljnPTg2g)e17W@aV>9_Wx&Na z9EK(id$rCR06QV%6Igne(f3!9nwQ=T*$gBVO^Jz#EvmQqS{4wb0Qs><7>IBRsqhzK zbrn=}l}8j7Y>R&{;42krB6C*9r{_X9TWfHs?N6kLq5rX($CG&(%20UTh_A_jEQuOP zhfIqA)6`a~5}HMGzYb=WBXYukAEi<37O{MjK|^>ul18_oS2TE03dQ1Ox}yB7bMFCxvtWR=7xB~&2+c*KlieYi|ibIyPAJnoBq}mU!YG% zH71U9LMNRpXj#G8Kox8bwdF5EXr69e7BMli&u@H1hkv#4U&eMMz>tAHStH~$u+nK& zN*WY?KAUEzzwTm^c6t@x-FAvQwgaDU3j{@6$D%$zAEB$Q3^?}wKDX|8NwHE@nsk~n zZhv~cwk7YTGOS>T?B##mHQ+LHSt~J*je)@Zf{Wa`eW4DoKLnrMmg#faF#+wG zEB~xUPM;rE!vQAFbM3Z>)^T9ZgF8nUBcFQ$der=NeyYf%% z;u_Gy^V;9v7h~;foZA>5n+{@&MY2T;O&v{!^MMQ&nh|(GACL~ndDgQSgKjO7$w{T* zwUS4Z@q={uudv{z(LQs-&(-8ej2c%NabFz0Ed|LV$TEUi=t;b}dBtkom+F&X2&&1| z`X*9$G=#WjOebd#BWw^*V#5ugZg&d(QISZ{Ez3~2?YTeJP6twqz-e-H^rOkzW+u<$ zet;2z@bd=z_-4c{*z@3T0_FYdltxwXG%`nvhEN;n%sV0Cm{~a#=5}bsl$E|1^53KZ zs5B>XAe^HdHQsNpX1&mZf?EF^x(};q66S=v2u2yF!H|)G!sOquHG+Xd*{`@z=uwf6 zdhLFS)K!QO;U^7#VF`kFJt$bh%`oVC)Aq*phib|z4sAroE?NgKnk(dc5YjmSz@*F1 zt5KC^X<2OHVZq%?Y90;StY!g7B7?aB5C{%bvKJwd%ltk2Av&Heg!-ME*qd5NE>a)5 za^i+SEkUeATx%ELZuUVS84R<*XaJIijHo~T1YuqR76@djg%kvVP)jm-(E~K5KFR3F z_UV)^jDMowH!$EpL3TR?W|mzEroqxBK9A~shjyTkTjA9^jKy=VUS6&}WWN^EZ+5w} z4ZP4sKYbY2TfJVE2)O8*8g=z-zan{lyg$YNL$=~ELy`gjtl!FK^UEKf|5vWs2S(|( zuBTd#g(&YL$;6JWVgqMw)kIINDLC(ke&GgOJ}&Qyo`%I9}if@3(0D8}17ldPfD`k`}77?t301tb<(Hj6lSUAn# z7-SKxn#p@j>HO4hLZCMIGAix~i(SLCW-7f3+%no%pQFv~@Q|v>@@BvJAOn81zr`tH z0)rz87%){R%)aaMdLow5Q&5qroH3pL{J{cPK=KX*O+g+CTzLE+h-AXAXz!S1qOn%bhapNCL#+hM|Qj&Z-=6ujjFZNGM{C1@UM!pW;O4e z0X77DXD|#nsRCXeJ^%U^eW#Z+f~6lALZN@OtB|wjor9+xjaW<(Lfr=KTCza}cV)$A zvOhJ%)u<50rN>PIfy}j=Oz+H6RNj~fl9&40*qWB?_WI3YP3z1k&TPhtQq@cbU=%E@ zN$@{XaiMZ)fHZ1-b^rd_Zxj;2oc$;(v&iK3-OsYP^;Wgx(j?GlZ^`&>z*q=XH;cIP zDSR#P>ViY~qIdK{d*#l)_+?Q0^S#08{c?`u3MK9mr~i_O@pO5@T<*&Z$;-)JO2-PR zlF0K$O@kkT>ZRuM{CdnLP{Q_q^-6zE)zR=NYHsv5AAd|PYEv|jFpGM@#0q;VTDPs* z%^nhseIAE!(}$Eeije^rg)g2vZH}LO2ISH7?#ouPuh~zHlH7Vf5b9miwBF~wJRR!! zJ2wgovPx{dni>~8phkX0@y#Y+>f0zYO(0D?sFneY z&M(pLB5jIv(-}AmXn|%amb|sK6!msEvhBX(Cq|yDlLkh=`WH`cE`Spg*eF&fwAk|b zN%)CtePs>7Iyy2sqa;OmU9UepOKHBUWyt%nPBF9HMSHOv11(CDk=247eqt!pQ8vni z*7~-wBd9V+06wOWJqLIYS5SP<&y_XBKa#Fu5nF%FyCFjqc>R0KC|NCq;H*V`i^ zeRMj84FCx#%6Cz`nI#%XaWk?l_s0U%s0bwg8rCO@8FK1MZ1NcfI~SmGn2(e$!}2cR{jNX zTip@s8tqvv$5<>xO6`Hf-9mrN-93fz#ax39fU9{YH(;j}ciwMI{qaxcnrGjY$Tiy) zx^j!d{ighy^TO+KiSy;uwk<^4*Wu~lfBo{Dx+XeD58IFGxGJzRysmD+J@_-<=+_4d zHaMHNTUC=^YKf&vy~o+Y)`CJm*h`wFS<=a4ks%!@@mEzNh^Ot?F}zqhMEO69G{L{H zedvi$w3u{W@p6%73MPcHpqha)GBUH5sy?S*A1^iBR2V(&go3(hwX;K}wX&-i9J>_$ zG*`TyO2S7@h>wH@$McuT1htE!Dgd~sW2wCJ(aeT}Rl}e%HbmS2E>4jy?I?`ta5^jm(vI1GOZGc67Fh9Kc;#YAtQ} zo>x-0lt>Z}q->%VWT_AFeoq|}0hON6q34dDM~$?DV1M6Z)YdkR{U=Q3B(Pg6yAYF( zJwO=Z{4umI2&o0h6b2wP!~dnq#VOj+3$@V&C#(B&*CKfcuaIt;aOuoAG>Gu?_rVdb zR)_S6isRCj1~ZwYohB;-WLJ!&ek?$+5G36-OM0DQSOxkaA;`cuY!K-}RN33g$ER@6 zl8`0lE$XXR;8I)2d&NNun9ZoUxk=IHBx9xgoniA_>2z=EuksBm3e-!5v9{+r0e&_o z_1H{0)~i8Wya@R)oj1{#(?GwOfRHYSz(K09T=l0ViJQ~h$L=>G*Lpk6$5!IkrRsk3 zyQjU!#dohcA9`pW9&#U3ErZU9I$RwUN*Pb3ROsdC5;KqwI@{aTXV*DG--`_O5L9rMqKVQ`cvykDa$^jJLY3RiF znXYANP>6mF4I-^I#5bWPVjxBRj^?5>*7`K=>aqnfSS?HqU7AB(XG2~7W$pLlPHo5eWbjq88TvmiKaz>NMdt%3uCf3 zIpl*P3=5hV`4bPB`vJL`q=?B#$VuFpm+|{{FbgUJ#QAhIcBE-hQoP6kHdqb`V|#~D zU~_bqjTV2T{83J(2rI}WuGeaDCMeVL5+SRpsHvTP!+jn)417>MecpWbZSy<$$$7sb z@nHON{OG=BQ+zgS)v@8e7xZE4$9V%*wm7 zmet350l#%(q@cps_JgL5yR-3^=U27it*LKfUyfsq+per~ZVoy+gpdZQa_^TU?x)+t z$Lga3Pfwrk>tgli7XQz}a_S!{OiAS3ew8$KbT4=?(kJ2BO(Id_XhWwfEw*FIo1I?O z+F_%<&t>D`fBob2Zthv0eX*$Bu7<%z(fxG;$IFU5vnuPH;>PWH-9zg2iG)kXs;-Yf z;JK|8!r@Sq_}zDQI?>_RX{*`oUH;z`sdZm}Lo@q>lZU8dr2 zoHW~7znFpbxW}ba$QHkds(^PAH%1c(Lo&V(&e z%D+XWj*)|_vMS9;K@}L@p!9{0q7e92b$9i*n==ft8l8-J9fav*hJ8O(j!*#Wfe7-r z`2wVJ_EyJ3A`B*uJ_jR7D%C|I;3GQ=VICEfpakupK{()Te_9`9Je+OB<@*O2Y%awF z%rt#)vIZ$F5QyhWEhPZCu=IRqaTK73q0kHM8hxj&@znxV5*5M(F)Lan#gCv99A*w< zdD}!50tN9Pe4k(>{Gm*x%gafhyiU7-<;|H_n_?|SUh zVneeKVB+E5TI)aVSpU&*&p_XKevv1u)yJq}Dx2r5hkqSUi{HmP9^56a!q6R0cs$?d zoHyqBJazc(C_4ry@mjl2(X?L17?b*qc%JWvw>`EQx7`^V2kzeuPzmEn8qhpVIU>BI z0*22czcQ!(Q~N&igqBs@YjsG-NSS4Q9(+$}jNGYD!eJLUF^%|ggPaT@SqL?BI6j}b z|IwdP_ZxDSc=|Sl>+X7@_V8?fwx7H5aJaU)3+RlhQEq=68o#3%j8%zP9DYX0e|Gx4 zD8X&FL^x|DKeG%DL0HP?#$KwKbG&x<>00(SrLlQm*LufW$>IQF1I7|_?`y~Di^Q%r zeRajX^cMXQ5)NM61l>1L9p%uiU1A4TPjH)NMrJ(^1vo+aFAqcirw{nFnrQuB_>quy zxk*a<-i?@N;}3vv^|ho6*ysU-a>F(F(ns4XJTw*2Qt-r2U~pSsXxYOx_JOI^Mo@O? zTTAMjfwm8hMQiHr{~o+8kon{{r`_KG;7G{2;GLUo!38=keh8lik?cRYN%HK}Ij;tp z4|Q@3#vv{a&I!LEia+QG3X)nY%uUQXWdg=yV@M<+Hqf_>Yf5^&d{O$k#bFA&CFZXf zEO98i7a!13k$DieODhYwuHrBVkE_HrmBQrBSD;GW(*74%Et`i(wvVVBs%%0MfHnd- zzw`$|0g~&s!=URXayrHg29ji-z(NF22UDXLBo83srPcb4ibZOvHK;XNvxKDKsUpJ$ zkp+VTj_#)2Jbf~xY0dJ2(x4iNx2PWlEhfaGCS@?WqVPH&g;5BQ8*~56+fZhx5nyQZ zG1=)NGJHq&OTV&b4ot0_uROSW%W7v!n$z;WY71A`R>MN((c+erkb_9+8RK?W>h9^3^eUgGUjF^o zGg|dC9#i+DLe}fC$H{tnIA|4#%z zpp);1z7Hp)i#InSR=OJe3D;pS9VLJNg>s~IAhgJ__Ae`94$vXhD})T%_9#|H^LQ;V z#4WP1{tY)^|DOI)yG(vYud)&HbUn(P^%TUrIh5IRFY;$aep1?1GUdheGTtipDIYg* z*!({9bKu$^le$z-k>YIM6bAhc`ziammCGs`v6rJL>W!mq38VX^n;Qw2m(>a?#PC)+ zhg}|BS6W;GoPw_V4a^u(lB>Qq6NhFwo)vA!$H{1mK^WrY9k*)3(_x^XPl?3V#O~jKNrjo9_F7o75xUn^oIU{yFi( z^1I_ox_%a2O?g$Zba}fm2qHcBvjfu&m2gr~CL6wt3$}^Ak%9ha@*Rg&G&La_VJ{P( zOBSo#y3EM(Acc=B_4YM2K4;G;M#J_??9Fj$WnXfJ06zxh(x_hV6Y{144L#m#Z~j<_ zR6{js+$(3KwT#ck763Y9B)Kw$B$ZGkoe>Fu1tUmBGm(0+(g+NtiFEjTVT0->Fg4j$ zX-Uj@XLkW+k5Qpag0R7%E)_7*())CBOz6<4kb;;E!WyzXZ$Ug&E|kCB1_03&8uHIr z@T5D~DL_ZRh@Ldx`Em3X?aEL`N(?WENy*W1gg3v{9B_~gk~ztroOfZ8gaUhK(iBK_ zKXTwPlD5KF>0#1^M#QF_QKZ9gsS+s+$VuAV7R1`xk|!Oi5Y1hNUo8TUfvU_HDZBb@ z_%_$G@)p71y+1nJN(SI*7_WuYP2`~}oA3%bP`6f0#$fLcL5JP&IaiE~MIgPQ7#Td> zw9ijZD;Um*=7wbl*Bwd)L4Wu4J<&WEdpw)26UR)6Qci;kD2mxmRJr(@3h z8Ff~t+nT^D$JM9ShSo205)5Wu54FY*_j@lxWsc8XBjRVQ<7*d%VSKHJLY2Nxub=-u ztkz-LwcmKI`i_qTtZ#ENRc=F0U!@+knFw;Dpkb48nREsPVl6+dclFGD_I)ZX&e548 z(wOBmX6dz-d;D8$-*VwPHT0=*XNn>!bAi+QrfKhT(u;BVaX-HYJ8)*+clU4J-jUx4 z?y^5*EaJtxV z^Zn3K{IL5%lNaC`xOUWod1ou~O+II0=xtx&w-fOyXE%TQ-+pK7{@%4cCy83aY4$u( z=vT8YOr=6zk=+g_cKI6Kb_JXB3d25sPTG7ub1fQH{pQ#O(87u$jWCpzM~;?N_bwLt z@B7(CwOg8yP?l#z!ddZH?3U`;-*3TSC+!ywDJCZnvN+OEZ@liy86CP#xF{+y+Ru?v z*gLM|0#h*mjrUTd%tp=otv^oNXz}Y#cu3f^@fu)oym`q!h%D7bmzc+udA#qlwAM#V zrcYtV_tnofazx}znqcJgci0SCIn#N=rkGvYvr?wO+G%hJsI!fwpK!Y<1um&|KPqc6 zJ$-RkZY?J?G&~%cv6Cc)A+)0<`B|B^FRWXuoI-{Ps`fv1PQ?)?+_E%;x%@sAn1Yck$mM@)XaF_P#}Nu3Y@6XA!#dmiU*obH%LrAf(e-ofm@vXRmhCc}W51-En!@yTO31}xD>3)6#Y+6*j}3#B zgsJ6q#}|>&!1D*sa+)KYm6>#(SPZA{ zv8%WPEyXm8VhbMk`pj)-2wD0i?={WC-+`99(!E?FGr_Hdi_^)_jH@b|r>mZqogk{` zjM$BMX15~w*|NfSGK=%`%k}2I+Yj#k6~6Vsuh2~8-2{cNB6xeGq1akqA_yH;$T8zB zeB8{hmhDIVJ!4>k5VYt~ck}nh3A^6KcN6{J)D18~BI2FjCVf)|O3u#C3TmYO@e*}d z@>VJ(1G6z;I27uZs55Yw3K=9RnTj`ZvI&6cnZlKU0`J_{;dVGF>kcZ&z}mXr1+F3dxKAP$=uj;7s@VQi`;Qp4V^=7|8ox2>hHy250}(&l^$bDT}$ zFY)JFYzT*snmrz|B`<(cuzv$b$pK3qh|`5Bw5K4ukknkk>dP#}Yty_8vjoUwf#vh@ zcuBx&akOk1qwqr*eoBHQfte*_$MsXx4j^gr+WO}#miF$oRrq(ycatC$Uh4Tllbt0lIy5Z6R2?Kj3Xk|_~u?iY9> zcTqaj|2@mwd~sps#NnS}L6YIZ1%tg9JWFiOt&*SrA>sBE9v&Eb?Z>v4878SeYD^EblSFWtp)DoR-il&tpyIKzHAncSGJ!q7K>f|sWb^x-ZN~)J-zh>ggdU~ zhTqZrPen&ULGxputluZdy`8RnKA<_8ulBfJD9(1DEnU0+m5M-oChz42&hDfdd*7_w zh;5OnLzsw#PZMT5iZbQ@Orq?k*v_3cd3((574NkBA4c{0FsR>+t*Cns@32iU@D)2#&2n-Or+u1 z_ffGpOiZ94Q@xzmyf>2(jk0Uxuou2 z^sp>leS;|Qiifq0ty-J{;W$+K84DJoISsYtLUC!*KcOnfo9idRLNyqPQbVfHALMZQ zl9jC0z_m7c9O{qu8T?&8IYwvaG&vo2-xoH`x-ZoshtoGpM`Ldzb_7#WQp}>kyo*BC z2R?32FKFvBK?bEXSlm8u*+Ab2!j@wcmW`xLAjF?PmX!lpxlth;#1wkw5M*U2Okc*O zel;}zlb=BMa(brl4^S68;wPktiL#^$JJ!t|2`sf4YvLl5ok^I{r>#Et zDrI>Ihp4SLSFA+g4kFcj{f>+eMJnTNB3GcJDIn?rerU{dk z$b|4X*(>=*F?z*b*Ctz_hS75OR~$yB{^Bx(W0N9v`=<&Kh&1)wE3w_=Wdu{09O$Ys zH$_!7e-P9i9nM+#sTwX&JcYv6>wp6aM^fu8I`Gele;aM;|DG2rnc`~QIDo=XDx;x| zo{X9+MaYwA^~%KIqgSaCYeu&s#;mdl84ypq5Z`Zmc6{h;2~pQw`=@eU{*4Fja;&%J z-a9&ab)9fBR_V8#>Bn~GKC=Usop!7QUbuf=5Op_?wtqDHEljO;@jvGn@kCj-jk#-m zK!kXZna?*kp4Qv5I{Gf{P*6-EYISAnKXY~e1A@wq=OTHrgG->4Sc(tZ+Wo)8n>B8c zi*r`U^NGM}z+6n`3VfIMmD9a>O3uFoG4+7S6MNr-><4l@T`Gp{mtKm|>LbC@=aXFF zTAS}H*HnM<*8)$~N89{0$e}snJ~hAj*0Qo$&<{J00|U37MNZoO<_*L&(M?1UflRAP z$BV@S=PsqCT4VH5ym*Qx_onO}ml_Q_=ZZ6_vFyjrrT%Vv57?8RF81G?c#g8Nj%0eS z%@tK_r0EkLe5XwiOB4wW4Xu%#uoG@O|2FPWP9x-RIsMTgNWb1n=2V|`GETP!o7C9k- zbm~UiS^rCBI+V~gy7UT(rCa5Q4Uz(mN|6GClfNo|bm;Km%)_7ABt7u09CaCHDEF^R zX%unhVg@S_wM8+bB}{-O%vYT#+$4+LSx6sFCGk!3y1zQB*%%sX zS?H9@u^1f{;*?K^#5?iZEQ>Kum7_9EAhZi1R6IH-Be04grF0^X^9w!! z#y=|kUvW<3=P?|)N zk4_8H|21vIt*yzNURt%CH7uGQDqDW0f7b1TRP}j3aV4S^*WjI98tj!>XkriXX7w-N z&QG`X4P-~({_&@OQ7;cKK36wSW81Trus-i)}p) zalQQarVcl;K3;X-PeG&u?&Uy)vuE6A>ZyXp1FnlC9%^qoHi_1p7p>c#!ZAeD9GGmY z(-j=9>JQ&b+zjQTpJe5pH|{7Kw?5=qow9NIj^Q3ZoV_%nOSAoxcvU6>i^^%+Y6(0j zKaZRM4Gk_=X88Z>TA&2b!*Ye%FNYhRy=~MBzTe&L9<>{sergF^Gw+XWE^OJVs?3El zpOiSpP3=tKE+?D_%3d$$KZq~@C~ylSjs#ggSZfP&(d0JS^XMP&sWR31W{g`c$tQ#B z>J@the`FWWPb)Dw;W;F>G$j(4yp8fn-IK42!j?oaCp2d^T`@=~|3zO&3Nmp|s%umN zkIBDJg$<3_FSSUXph6VB^e0la9C(Y2H)&7dhhb|V2qQ(asevLvoA#yvhp}L7rQNTL z&b;V^Corbv3rtz|gl4tcZ=E2m+2?MpZfzw7WmETZ0r%jNk6#AiX>SV&0gVKSrQKr8 z?m)!Gq>nn0S-IA177wp1W;`jvU&dYi35O2$X|53DGMN>TmcCc(&1;Rt-Yo+%giT3f zl-Vx9U^Z44OuG4yZTl#PZ%Ks=cU(=@aZ2~*EV7?D*a`l24<6`7CNuJJ06-4^E6*J8n_7wg|9yb% zRbsL`wT~^?Li-AbTJ+!Q&GSsVzpwHAA4i&)4E&?PkRc`1v$W#ldC%uR-s+#*_EHvJ zsjqtcS=joe%AcrFlZfue2qkh{(r&vo&N(mGJE3F`XHOYj4{Ut?E+Ke7VC?Zawu0JS zZrSsAG|<`X7uNF8sf5Q>A7|h!uY77ibIY@n=M$z68rAci8%5C4RZh6P)#V_8aMtdM z5AEQ=_3;grsLoH!cs9=y)e+~T>A3L)l-JmV5s6=c3jAe?1lm?P!X7j;SJ*IboatU2 zOjTGKoFBi&GNlyPB4-$L*l$WFmq=;4cWH%Qm$4{(T16&Fe<31*?33R| z)X;DsvU3UYy3t%AVsJAu!kPls#_AQeon(BEuJILP1Qit$QuZ#C5o*cEPQsyDZX^O5iRh*#RVK_hv z12#+vfBd*ZsClizVCmq$Vae*K$1A|fL&jtVLP17HFJ;AaCi2#yNA-+TsF&$J!O_O& zuI)yPxG~H9W5mQT#z?W07L_wn6*t4x6**)}hxZP#&kUE7;ch{(;Nm68>oimRT3sS< zV2BdVw=Iu6v%vX~Y85Cne4j&ceS>==e!sn2wlKQ8)^?8GzNhMm3>bSp7w~&okfyr( z?>09zDr#QySXx2gT+r2oEXP;dTE>L-E)!HWHDWCv6h{xn4jz>o{67fH*3k%>9CXrx zUjC7)Zif`x`QCMi`93JRm-`kcH?1X}zV!7tKHfeqK3uJ*t_+f$CvgifFIQ)pw{wf# zHnrrq4R3kc+r9YQ#^3$18xzWRRjW{Y&X{~V(6vx|*Z*zwVz|D}&U>45hR0}4qqvu! zMM)QeML3uyiAOC7r$f@#CeWP|{zl6ng`;D;X$}K;qdeJ2ji*;f7Tql@-0;Y|QsXd#64#6k}`tpKAMd zOO`|gDjzo<0(T?#<=gS!qf4qX!{Q$PKA}6d@7A)`Ts9uGR&EhI^)Nr3@uwQrHmp9> zEw0nQ$X3RW&I-7;X^ymLzs)OVHI&{dJ0DyKsq|t6m4aPWu<+G`>k&Bv}>$7GB zSoge5KXhhfhX9*`OE(1*Q}S%p+Ld z*oBw`r;D)j*iWFdif z_sjkoq%|h{b;22#T9va3MPOU-c4akRx5P=6c=0E z0R8@fyF&GKzzpJYcsN@kV9l=Og+^)YH?}A4=8riR|JvAXQg5tpzJE{ROy2Q1M&NPL zQs7c||9Jd4kTdW>-S{^jeE+}Wh1a#wBQNoMuD<4f^8_aHI9;u-sqs5$vN~FAGq>EK zS(4{J-=7ugrF7w~r$@ws ziV-^V<-M(Cx6od}pjxH|z*enJ=65Nn{j`uRQw3Cq65*@<{-dWa&s+^40;WXbfzaEe zF#nmR4Eq4GQ>lifeG$&)U}smvP@TElj`W4be>EctD;#uk8FM#6O7pzH?|t1lE!tU* zE9xPw_*9pqk{pJFPn`0%_pH5f5ne;CS=ByYGVy-*WC5-tTL6QZF@ffF|MFh-7dWKux9&f%?^iDP0naFVva>3dnOpQfhY?vNr5p3fCys^Q1plU%R4Cq&WH3n`pA z`;#krk#upt_}cYl{_$XrCg4JK|NEPz$8Q;tS192(NAq`K_aPnD5>HodhMddi-D@H* zH`hA8?YjozI2csls>X3E%I^Ps@$)k0tue`IsV|dLPKPpJ!J1&xWHMqhn*d9{3Kr@;I5|_fuNx5D3>wWY*i_SSWmE8n5(YwUU*$;vZ)O zD`J7w+9oIZg2iQzTh}JLlzH+%m0p)xuKnMss{T8Pk(FuQiUO%P0*VlRoU5 zju69?dx8^e-vCL`Smi`a$AoTlJA=oCo=wo7uf_a}mfEc(wtjOm2Pe-^QJg-HuHE&b zw8za~7wEO`?it@~jZ3(S!gv2GTV9QA+~}7@nGPL$IZpzsUWbc|{}2g)SE_55^F}T8 z#-1xN|C6WgJ;X?Y{83`w{~iTBeE+`2>!`slaJB3@+kaAtm)Mi^sNs9-4&vup7z2Bn zW*pj1`a^Gyj2GMQ{YN;To9W#8MW2&h19s%q?&8D0JPdqb`S&r*Xm7>Nyt%p6&*`O& zPR3cw_0@PV9M2>&C7o|yF4Z%qDORul;;{IxQ+Sbd`d)sF5rl8r-ZVHH#uU_RG(hM2r45GH&j z9%Z$K75=@=NM}!CMI)FrZIRX&uot=q1d^2FV&;09-woAA4#(g}*zC1DktppDtIn}< z1R!qhQ6&pNjZLZ!Ccc@r!JKj#AC(CbPO92G z9m5Jz1&rT4a;VVFsE%}zQ-W97VEi1l=Ce7_tTo>o;|T|RL@hL@v?!uOHQvTTDab)V zXJg3`-zYjw^PI_>rK^?o!t;?OB(PS}VW9?N1ykaqaOcyfAYc^zG49zry|NB6c3(%9 zhNa$H?b<2z_Qf;zk)YSSN}O((771&DcJLL8cPto@ zh2Z~on1at-*<(dh3p9^iFCPsIp5a#F4UTK8xB+LCf&Z9|!7`TupZzb}mmocCVuE8& zMKh;&9b>}BlWkVNF0R*TZbw-mdQ`vZ*Me%D=iW5H-@mv28Fwq(**8Mcu`}r?DAp0!#PG6JkGrgX%ylesTV1g7oIGv5sP94l z&Ll48OwcrMkQ_hDBEPk=HtZ6lz(Ofs9WcU)-$wwWV z0aGfqU8;xKxKvR=hSx@>0rHsLD!jBrQsVhJn+*M$EI33=r@VRHIOa zG`)_u4DE@Aeq~I0gKU=)0WkA~hMoV>TH(`BwlB-Tbg-zsQ+YES7e^%*SERwT(@3R) zxH{j_5&jI%;C9s(NIVaNy6RRHdt?27OuhAAl<~Vf42XnMi=-g2G?F6Surx?6-Q6Iy zgrvmM(k;@kbazOIbS_9sEhP(xbUk~{_k7NC{(}3L>vdl<*UWonKaFF1q}bl*J%7;6w5v{0MjibV{ze=a6 zhA)1M4q+Ze8tdZ;z6}+W78-3a5k1tL<6{6tV56Heowa?jSB05EX@ESu;vv8HQ=-wq z`1JqAm7}FF^fRMfECM||JqbaWbvRkEjp00ohRLPt^TvD`GG16VZ&%DPbzV_3)ebk< zvOhl@r@IC+x5t$Y+I9+~JF<_v5)=~G9P>mHp$8s_RsD3&{B4b3%D_S69Y~G-MCX;j zFz$9cR;fG$_PvLQS?s>`8=GZ%dhFJFn!XNGIHB~-{eL%L^S-mhhdaVIB3{-{kFsd% zw+m)#J%MRZod>sd?_ful%zSM0SK=2a5L_>eCIL5ndR$WI;a$pYknh}ds}V;@#=Qgy za&(fOKc?>g97s}K7nB&&+@!xvM4MMT@A593)*+Q4TCx`$=iv1o%yK@QyAw>PVaZD# zUqxvI@@gp|2cLYZI@+m;2dMEvCR4{byO8oX5zlCWS7rtXzH46tD~n0 z%0%yr`*~DqYGm}5T3VT{q*&NV84qLTlLWVuH=rN_SyYEN#3>sn6pMvBlmaI9lPf-z zCJw{=_>C2`%4Wx4E48ZPM}?YoNG#GrE>?UPDE#kqh+qLd2bWt z3YV2sPg*&|Fci;|PmqW-5p2R=bklcI8Nr+?6 zh{zvHex23*&bif0{+t_Xx}(kISsEfzp!4l}p42WN0XG$knx!`t`S2@1(v0ToT~{pR z)AwD04G7$=`<9LTu@SOb`SiQ6%M0-eOTg}b(3Cr{dcfr!WiNvN>C?%2p0`QXO?CcK z+ilFV0h5Ez=2sOH=fOMksH3Xqk(q?h!;v7(jT`pysoFDV8fRRIq!VbkzL$D3MQVBH1CFGue3n%p98F zKp{y1FkgysTI@BzVYeDj5dvh_!>IYqqxF+Ul?1-_vg(5vdHUt`|hv zDJz9~F&>kl5$GRdfT6`Pq@f-Q9)`+haIzzLRjjvN$~hv?FYagRynMf1R z()tEQE-$vin;G z8ECxlcg$@+Y&Hk!4x}&!w<=G??!qXhKWVlK-1~?nxbGZ7f=Y9fQhG0pJ_JgIdN*0Yq z$UCpr-PL7WEa*N4+dfll6e*$v+dQlq+5QG5^H(}rXuM$ia#tdl#fGL0*7IOzQUIyW zVt;LGhYi8#$;L_spmMMd$_N-nR0m~fxmtBcqrr!R{h6YZnHt4Y56Gd@vm0q+SAv3_b^ zCYoS#X@WJ^cz?llhKI5(Gd&3)g?l-Vnt+8l^K(S|3!u#Sp)L-=*r=-OC`sdm^@P1c zLlbHxH!JSTu?CKbjK^o%YaW>@Rz5a@KdBFnM|@%H{)6!j$i<;=F&D!KbC({Ueg|Sf z&%CR!0%(yZ`Om+v99w-KB~bRt)>DqxC=_QX{cuq9Q-^sk+ zFH>G~>Njp>zT$v>9ooZ31CRrfpRKOcj-J!8(8P=-QaKx6eeGrI#!O)mhtQYi1wBrg zKGibUnTWnd9(My=v> zmQ4QV7LkDz-ixl>6?k&n%fH!|VB-K(;pnt|Tvc6HU`fUb&ujZq*3p zUm-I7_>T;<&!Lb{#cSz__19B;9VCbB$N49#-AAP>iRJ=oRE=-E8_##rid_E6YBPas zi7L`L0BuGFx9Q=g|Gv^R2?Gwb-VPGyp=}buV3hooV_NVm9>+s{e$u3HbHrhwa4OHU z`8atz*A+5X7@f@Gq%72e#0r|7R!n~Kub#sI9hB7bF;{Awt%4`RoJCMYSg(LRi3LmV zyd%bu0LWsCYs*7apD8PY1D4|`fHCR9vbC`aRMwnZ+J#qm^i2|Fpo6Nh)>#s0SUl%d zFeGEaG-jzXouzWcwg`!n&eW%GtU2}#P1dE$J_^Y8lhZuNoii6gm4o92ldrXfFV5B> z)3O|>_*W-%m7^WPMtL!)3@GQ}6TL+>sHY`xyIO+xTW&s)g^LGl3bnDO{-#T&IO-sv z$HX0;wVN*6D}OBol=dWul|PkRucL}d8dGT}Z6smGmQtq!WVCRTc&3LgDGko!ky)W! zdQ6zI+-rX+24=&vm0^l(DU}za6ekas>w9ZxA?;~WsK&(@-j0A3@AaI&aTVMe3T8fF zMW%(>Ap#G64SEM^RT-^@M3qT`4Etl#ylp&meAC@+%rzI<+v&C4&F53W{p`*Ud~Q!S z`PYkXZH2YG3W49Rx-J)kZ&<#19{#R<{XcGw+;S9v_B&9?7o`#0tqs1v&3U~2dxzyW zNp2Q^eS1kOvb*%;yy0777TFv4V3MI;+vWJ!I4`=M>=lN5jh$eCy2mHGU`4+*u2;|p} z$#E#k!l0HOj$5(-)5FfHa9C;SfoFC9*3Oijba7|{@`q%~Mtgn?bEI|DUgaDAPAv@a z<{ve9!&slJAu+JQALc?F)^FHl#CM8?Rn`sWwEJL~Dlo*(Z>kdx_E7dUzj z65i`b7&o>hMB}Y*%t~ZQ5pznU5%S%i46qlejn2!2;-!1qvP&9adRRmn5LukyVq-Vh zK=j_F`g`!DYMXyRs_tM~&|^pO)X0#_IQ;<1|E6df1A!2l#naS-V%c#9Ab6&)=98M7 zPKt*vU%H0j0DOEk=CS-|m@-a0>>a8I^+(%o0|r#zU%7_WGq;cR!o-@Ol)_4xa!7iY zd0bkCEPDIQ+Qr+DU&M?BqYGk48ulZ>yO0ksPOLI7QBScnj)u|n$rQI6>YP+8yC3*z zNN(RXu#*%TQ?1hM)(qrozA9Ob`DEWF^40uK^6cwWT9lo{c9-bWSYr3zvZpyP_AN_B z{ueggk1@r$duYOA#n*;3N(~^h@rRwh8J&CK4tJU6**5;l)LYKM1L6?L` zDsF5vL7&#gr=yckC;wn87f1_R8t4OL)iYi467%(G=J3;I?$c>e+5J9)Y?;{K@DF&= z?=l(vM4g6P1da^zU6C*acokKam&>RDZaDF)ZC&M5WiQ3=x?k|ok@6fQjGb}-W)J=s z^>cpWs6K`jd1>bD5o;|ly-p~T2hJ9a=d9HT8_4E?&{NP;-IV~schrHY08JY(ceQ0> zx|(If=Nn@KgNqy;Cqbp+D%J=v&UiluM}cCTPBp8Z3=YAIdiFs~-YET#IB3J{jt76S zI&PdjJsCe?r`Qtn%2-${KZhJZSS$)9MD(H165O;b zNDUWt;lICR1u=|2>}{EHg~IifD>B>5#;Nh@QYwV$`Vq{;4G#1_$KQZ#Kj2wvfi2kD zf5O^H;sICnvVlL;0BpFQyV^GEcHP3VsrcGc286I`j{N0DIWOyZouRee9Y2frN0omY_Tr2(XW_YpABV^*QYq!`(^7l z2am3Z7D34;2ZfHu4T#u<+5K$o!#!cp+~s#G{{PVP?`Lhpd3P|q?kcgK18oJ#iwtZm zfjt$jyNulkNqOW$)(RPBL&eucF%5G*PClMlGjfOpHbp zo?-9EuNozmpZJAW?~1mN`?G3o74>^C&(H!3de05Q5JZ;XtKC0LzcMK>pS3aGzpH#5 z!sQ*&ZVrG<31^s-3li?jnwc-SxYD#=J$6Z?Rq=RwaMSZ4L#2j1K|7d&c6B*dplE|i z16*Y!D8h=^f|+ZXnw|`RFg4ybU~_1#XLDTe`6H!^Tj2g`_i!8Ohy)4eD3TDfm9j&8 z*rP(WUVDZt(e@$z+w>;do*&}quOKM}}pi6IJ_wx%|&|ifP zX0hq!!LAx8xJ4>wB_>rteghCau9BuBFz@V_r^49w zE3ZO{;E9c?#WM98ZsWr3TO!QAjb%@N6u$mTQus0<%!TuYxZ+STJmKF@FDp-xu|G0i z%QY$b#_&c*dWQ10N=8+|GE^-BhN6trObJJJ9wjEEoG&)1Qy7qA03LeN35hB-c+D>( z@7m7@l##;yXR>b$9UPBQEZ#S(JGIzNwV_;5KP(+bM;Z*e@UM9pNB5>#*UGfSITt>&Es zAKm5$3atJ6f;=)Zz3eK~6p}vsE@a^3fWgS`|;>>Lo;P?=)>h0@u6*YL8@xr68Sk5IVJNISy_N@jqMee z-XBmaM!Pu|w+6O=wcx~ z6xs81>E6AK&~*Jk%s5{UgOk!+; zWZLplyxqefp@{<~p-Ti<`MgZnFHIajw>+%_*vgTMH{^=-}t=?*0 zjsxnL%0H%J@c-jxw)~Mk4`Vz)dwecggmkC4{k=O>kV+XOHLw^GwYj|exX{{jah4c- zk1cuqJ^21lg4+{BGKc_ep1kWmV;wTKVRpHkkt_axhvW0G#N{gAi)>quWfVQ*EJ8J2^-}jdU^j~PFy_*wn}wj8mx1a zE2w=eMf2G(;6%U^VlA+%nM<2%q#F%DCZ_?|3WYszVgz4v@+_y4s|h5S4r2I3YZcOs6M&7O>9;-Seq8ZV@mrY z{&dQk<~_A!w^ki+IG*kK_*hn(C8e9ftd{-WVYZl?{UR{vW~|p=;Hdg`;_13bfi`ng zH~6dV%fQi>lgqKY6T`t3TmL5U@=i_A)PrX{AI)g*Xx&2d8YEtwln>mT){0%|t$SZ| zw#;S8fP?=8*22%$E~3}gc1IB+P$b-6H~+(@OIBYl&}$4I@6GKY{I_F(xfF&Qr zs-qe}0;5=|PnxW8;BM-W^{AoY)zPz+*bm;$SK(1P8Wu;qbm2m%V z8YHo4{DTbwp&!{=pwto5O~iQfv7oJlv~!Ur5$F}Hsyi*sY<^oT`*9%Bn!e}TE$FKqF>ykufnmi*9zTH(3r!ODd!ZxkoPTu z@yV;pg#Y`IOpFgvsHuNfq#(o;+X-njSiF$g;hM7Ld+`Px^pb?0m0DC|f)J2|SbVQR z5~8MKZ!96;2@sinr+vJkqlOb#3O+YBRJO$|(F9K@)ajUeRdg&>ZLG^He`_)(ea$FN zHxZ`kk00jz8+P{B{e?3e2$D%Y?#B$32PaM#e|Z0yrS2eEIvmkF6d!WwvUz>W@b&I* zM$mxd?RtLa-_Zk;^QObVlYotQl=e9#ekFFdG}W`4Zd7yMNN7A6G(+{YNUm1Kj~Wgo z)Rf#>&ZT_y8rrxKHM;_vU9_T!x$phGmbDQ{ku>vTKF@1PSkn&X4#@UWXum`HZ!GyQ zdan+p>umdRq0tZ>$toY=->6X8^c-nvlSBbpf^FRRr zTznN^v9PR!Aq8HmEHBMHy}A9h|4=p7k=kHLxQVH+%vpjrz*DtgI_Yzr5aWSq(q6q#DL=W8JL;hrYN`!X z=t}8}y~=KylhOzeRXt$DwTH%_mH6RN7E*WPVfW{z7WaSaY?h71al3XGq+bE5e{>3^ zJO5(oSc>Jdgr>#@F==%0X@TfG2hgM0G@Tg!d|uuhhIe(jj0R(8b*}v0Z^%$)Er784 zK|uDyHa(!}81b?Hhb(A(`At+VA5L`rpT!J?;Y9x|>Q>6ZB_sq^XsrOIIrj=d)1&8%n3rp9rH+c^dPpooyeVQ%c zSyXbvO6X11`0?NwbYCkHnu3%#vRfUm3#qbIwHn%)*YW^;!5Bs?dJ*F#z0aJybdzQ6 zTe!3o7&FKO5z!!ZU;Lg%rBg9hEErC7TdeD(6-QM#YN@fc2ejo-7GLC%;6P41Ol#7C zg)rnMl8sGnqFPwG@DQ0j+vZ6nHR#d)zOjR=@#bEqBvOiPV11otztJxvm8PY;ThTzN z7rs(orOeAw9NzjZ#Bj`$)I z#-oA8#BYO@TON-4oah~2bYdMF+GeEoPbB_F*zpVD=?dTLicisB;xsHOY-L9V#db^B zLyBKmd}+MV-MFA*FlIAILHh?q<7b6l#&DF+-fELjb}CY>wwCG~87jVncn;hDim%8* z%=2&K(v!HEl_)nD+CX7#!i+kow+-MuaZguG9(u~zPbKx+wh7;2n&4mY7scAXQw=d( z>#nBOW!tr9(Z$F!xkb)A?@R=MUS941#1Ea2dQ9Z{iJ;#@* z48^UOx9bl`h~!oKRNkcZ|4vJ(Ue8}9vj_N;Q?JBr-|efXi)%0D;IlI1cu);0qh#y= zX>|PKZ_$yucw_LWu$j;Iw>>8(d4Ze4dXjtDuIu^1H$yCm)K9&SS68172WTVm5A1I` zUQebqwr+jp+qA4%Yrovfzdujp96g+xu`(-}`weth#O3^L0fqDY3*0!Jg++6Q)r~~D zT-yJ_ASjx71`5GJU(2OqrNgIy-t^>cce1p!lAA1m$bG5c&4!MguC9n`g%mA53G2Dm zWZQ2$x!RMk*H*O5%U`N-)|LtCi));I#523lBp6Wr)l_5q1v4_aqjPmhEL(Ut-}3zr zLYKk8kvAP`II3|(RCn_!-9rmrbL6pj@xW2SQCM8O8ZX(nGmv`buXo+2uTnSnp_qK6PzI3-Mh)?6cI8!S zfD%YC?FW?n3(6C>%z+cL>k*M%-fOfeZ9Ph||5%soEF zc<4J4ImghUb& zc<iI{i$WyPEr7>*v2fN=CNFhefmC zHGIJ?cUju+QLcg41p$k5Bli*hKVk{J`Pv&-13L2U4=S%XH=Y8<2)&|OAY7uNSAo5m zNx5#ElimONJ-IvWd%7>$gz7yko5kwyC`3u8TaEvwUrC-n*j43x=A$o2S)xoziJ=^L z+0_Q$jRER-Ln<}YHY>?~I83S;b$^Vtr(&S`0kC!VxUMU zQKtLb%$ym9LRfZkMxCQ@{B8s4D~)N{cU)qrK#{P%r5N?F%Zae|CNnf1GhY&17L=0W zf86mtN3%GP6xRL%!N!CD#>>DVE2RVXU-?|ACz3h4zz|z!CcVR_nupZ12PRU1YNw%Q^XBe`U*M-Mw}D zxtFA7L}20SnF>qrMIBAt>#0BavyIQ_l+Zuj=!st?tVmo&Jsn62CZABgt-MUyxa(b1 zH#lmcxaVz;9J9;V_`72?Nt6nwblE4Vw3z%h&pyijRm-EGGn(l!c;70v6b!lbayG^mE*l%+{~txFueT+!v; z%pWHHE~^!$fuIav>Q`Zd5`pPw%#|j}=!}|SN1a*|-NkB#!&c5vx$qEg*3j~1Agi@O zO&DG)j{0b-^}=4@9RYG>X@NtyY}E$r;{G=7E2&~+0I^HD^tOw_+ZWMIAtP9!gY{RK zEHfRpz+CS@vgRK*uCKUPcXY)l6qQuiibj~ce62Mte0Bsh2hvvo+$BlF4W*~$2=-Z3 zpeyTd*lVOLC#D=6fM<`E`hq;f7O?KX-ppLUlPfwGa-4;g8Qm5Qluz!fm>Bq*L)eKyVlh%G{$hqKQsVM%!wl~kT zNX@Ik)Fp1JK|&>3t41C_4Z}yW13~hS*A-`fTB_dHJfrWYZpbQfg9j&viJ05ZSccU;v3P+*^HilcEOi+Q=%tb2de)=ca+S+P!GGzo-P&z z1?{@+dLM#Uua2G&D7?{_{Q<$9%FN|6iybA|GlvW$`RdA@-rIQVM)*s1DqsA_S=in0 z%Ep23)_szRyzA7+)c^i_l;kaTYu*S;%d!!YM!YBRlikMTWRlS3zFD8sWblM&{^=wi zwvD$um5638(3|yeePUK0BUV2nGz3Sb+E&-m;j0{|D#|-<=Nr^rGjz~aV^B&&gUEP% zwrA+RZ_<+2Y=@Eajwm!W#J$xRU8?^-UH}|9OgddBFP9DLDvEXm<%(VU*PUz=9Rw|= zwVzH47Pug$mFQ5OtDyBe{U0whNx}H#{>mod%?V@?)hDkIC8?`Sg4z~u`U`<&hS;r! zPSS%5mOYJbq7Y>Zq{pH+XPj&jerT6XAjUg{vZ4T;Z|pc`g^s6+a7k)iMNIsMC-y0Wpvvn1%85azbm3k-7?J8ecGGlD0Lvd8tKR5paTgimte_J4nD2^wLk}QmAHc-2S?t5f(7)MF#Bp=f8W3rV2i9;AL z+02{lFcoaJGs8qm4ZjN z>$Nn8qOUVIcanZr^-&3N3BtbSb4k=38J_%vN-SfxQT!2 z+%!aPp@HTbUZ1o@`HS!F5YL$w4`L8h8B-_eef6Rf5zW8TAOXI{o+I4>E$}$L1Xjej zcvq`o1fG-@F(7HbuSkN$r}rEZAa5cdS{%0^V&%x$3xBQhi+Qx=>OT z|0OLs#@@*ABcVVS39B>ZE3KxItmTAw*Wdd}%`Q~f{o;nCe34CzcUk^hmY z=16fgKo=h{$%@3ymL=xbT)ASx#U(gGua;m+294)h%YvTwGpxpCFP|?Mf}0J8et>Zk zWGA3>nK}So?KjE?Es*7&M7>zYpExmDn$Xm7PA1$$RIEUhey&vEa#va~h>xf6h1aJ7RPg>>dx@4d zk(}r|Mc!X`k>x@U&xl_^r7c9dRA}#BCbgtiP2Jp*_adS;Is$+9{af%gyrAkmqe9W> z?KM<8%W?DKy2lu&o{2a7|7<3;B;PTko;|1D%NrD6KojzKx`hOtHO|%F?|Q?%u`lyd zp4_h(5)mn{@8@=rMj5?N;a*DIYua}#wzkPe*gliJTgi>wLRZ_gZO6a%XnL;Hkz#y= z$DWY2tHJF>@lkdEf$vnVL9-Kk7l~zxk5+E`icoA@6LK4<&YKTtc$H~Z>)~v68d}Ou z)2pkh;1va%N`w-%tFt#bT?y>nis8@6!u+X@T>pj|UB(FF_y|%8N*OEQGe*D+r2Edi zM1W^rssgdO&LbN*DPc74ZNQ}mW&IXYbLn{4sr52NhDm@c1WIgy0N^O-`kGtG+?tCYjg93y@s6DOAM6nAxds+ zjS;Ms{YJ}?6ppaWH2RT;qy_Z339r?PW-EVGW3^Fm$s*I9pZA;6xa|bcV%;YWp@8& z^^yF8-w_cNP>tF^X-87eoD?8i9i6 zXCnR2m_y(T5Q9u}pUNF;9!N_$O?M&}R`HUXpB=Y$y<}2!Oqj4F6|SMFT?#G~OSD*2L8%%0&>RSb}zg=jOV@sQ*6+so=SLDDk*k8|1{iOj!JB3)yTDBCBpg=1Rxt%--F9@e-RvQif!$zEWRmNzd`P|1YUkNghr zgc1J()DpmTO{oaKui+Nw#=BE>B}APdVY3sqsw2_`X!O=p5maWj;W^D}HiWrL{jgqEq_F<!0dY~mx3Vk}riF#}?k;uZ2SRqDn2+y9!Dp0bXX-BaNn5g3*yL%W_x@|*#+lK{!_|PJkZc0d z6SA(CQ1fxYA-K>9Qq#RV(rVo7Yb^hlHbAy4{Se^Jz~BF@&N5zeO#<2JhlX5`z{0ky z*Kg}2mSVCB|M-$RsUx*B)mHdbR-Z;{w1w&$Ii~2)@K0Y|<_lj{14(!wU?pKa%sxiq zLmYBY+q3<khPYfA{l=sayZZw5)yt%|uE6!N`X-OEHyr~V(X0|(rAt&iKmPt+IJ92I|&=*md z)=hf-5lXD4)$zP##h-EMT>iHCeoWGBJ%?5;pYG&*nPk?Mb$IoYYU(+l0QJ~`!n}Zp zwNl5ET#2Xx5JzNDlv%-wM$aK%p3ZoI_B@ngNf&idy6!3#?NFnJ%8AhXmiPY&@0??&q_*4L*9-F4;_ozz#E$QNV@bDVps^5u1(A6`Bkp21vyT{XQy`K9m>{ErZ z6E>3D&l&f>jkp7y(ubWGpYRm-HP(~_W50Lxf86<*gra@F6gM$Db6vteU_T282tW)S z5or$B8vy(7B;4HGI9tesKc>Hmy>Vo!czLCv@ti~`90*LEnu}QFNXE9LP@YukEZ*Fl zhzx-d4E)w*+N&8`5jDb$?1tAFS~YfW!TXDzi!0({hwzX$R5LGX85}u!H_S{sRm;IA z@@Nvi8|&QqwRShIU3HhoKpkc7%zn0A%Vg(h{4m(4&nYTyQ7?`rnW3nK0h-=y;+)wB zNRw@Deh7t1QH1+>ec|ZG09S94)aXSBe+0#`b$gdG#s@O$Hdc*ve_!JSjyqPqBz@a` zdFkWt7nI=0aoax zB>f2e@ATgyS}DtUsVTUCu3+$LJkE;&3;^`ox0Vjw!Mznbsybol%i6a4yWp+in#{hP zGJ{0Sl&A5j`|-CZmFoZSp)KM~)LrE?HRt^7$^TEc`ED7s>tCqAcJ@DplIH0+FGg^zBlhANQ-TaZ{V4!38KKhIuUg@x8qGQE!5-Gg}+*>2p@%py(s?C z@H!9^cXn-cZEcg33ucmWj)EUHoaU5~aL1+Guqu)KiB-!0J0X!&L#LJ13mj6Hd0;_* zAg1QCnaCjJW|1o0qZUTl(K?!r-0`(%2|rEN*IW6_oGlndnP8=Cn)TO3GhZaCpyy=$-b)m@zPtE-5)vp-pKG(^ysOMF?VR6B?s9VeL@H_)j73 zZ=%WwtO&sD><4V%1iYeNvMhW(1S(cH(Og<@mAvRqaiw2P9d4SLqxsID@SdC-xNM$+ zh99zv!~qan%T~71TVtEvzS(#@6nrQOlE_)dQLp?EI^x#08JXkOXJb_b8b zcIqd!Vsv-h2VC~3zUvs%%YT($gC4w5^`EDxHw)*Pv}?JI-PC02MNl>oWH1WS|N^~n_DPfd-@=9R_(jiUei{5vHe^!>4 zb=3fyn}sjp(7Z9=s$ioG70~$VIIA)d$$QHVg(GZD5E|dhhU&Sv5bs)==7d+0cp6&3 za{4R{9w(SBQFc}_rj%Stc@Azl8if{tT>3b7`enwJb`|A!NXIB>zc_HYX5P~|C zI!V4%V`aujR6I6a8y9P>Kc1S}Gk+hcGc?>>j@3o{PO2>c4SPy_FJR-Q#jb1rZE0)Y z984i>B>1o_=&-Z*u9Jo5=D+nq)T|`KP$D?@V<@e!%iZv9^zCCX#HGya>XXF^-U3BP}#H0GNyuj|A}b?W&*!Q(rG>_HX1vOKW;Wcqtq_i0Cf z;Tuj_wj*>a{!QMt>kE&IdZVY)F1`E*gMfXvQwzdf*{4HT@Wtpr?fq8UcbWiM zmXLZU*>V^(+oc6WBwg;fc!ePFr;4d=#ye^-B*1fc%Xco3=WV`md5I^mgBgFy3hOYf zWiv_mVqVIu7^+Pg;;5{|AVUgM4(}wCogo253)mYgbJ{c;r)#Bm1`OZ-`aTfVe3IJG z-PjoLcgL}rWOpuXxpvQ_VVZ{np$h)k(I?rJ-Qq3VZ|$+dRAs;|~Bn@D^*d5C=w9i|o28Y}9~ zYl7VGe7_^e?=<`*@DxueB1qk}iKwKtLs6o&=?bJv6DYj8LT7>lFU;|S78vke?{mdJ z_>pBL_t5BXMr~!I^P3*I-pt)`G~+kIN~C0Ohg8qID$~PW0K;I(sWNbqO_FeJ@JM8f z;l)E7Z(C;?2B?iQLj^)Rs!@52L^|e^H!e9>PPIFwl~_=CAP+Tev;3Rg6M2v5iFmou z6YI1pVcE}({Ki))E5>50tB(UkO*YDk$)smUvb9J>S%z-u#-(R2-xzvg~{E^J$imnCBc2L&MPeQcAfZ zl5J>0U2{a(_4fT7>#~Sf4(qxq!9dZ3ih@>kzTo>hbdBb;Vr|!adLCbIV5b8fPvV8U z2Tsmzb+`E1N+XA#-IKG^OH?ouLZblM-cB_EdsaN7zq(cwOOn^gpqYy22<`NgJ;AW_ znihCeX@aVSSfgzwCCh7o*}{qyN943dt4fYfd{G#PPO2{JY?BN0e8PCR6pV&9xF4K! zew^3)q`vO^(5m{3t4yyFeyD~@Ee*3BGl+A=&5KoMXyw)7Fpx#aTV^g zr_NcP5qJYeftHgL+IuKm`SekJqdVZwAoj-9W6JOcv21b&_+&@0&(v`+$JCc}p4%gP zuJcxtvzA4(f4AHXRDbFPy*_CxjbwucM+#B}HjXx}a~k9>J}$SVUpUV!9_q#`wOqPQ z@bS|tUDsux>r09FINHR&wsjpcpCqA^>0`{!?tTSa{w?DopJ6p#?C;$O?1yGf*beag zYouJXX*NaGGX$>OCD2Otv>LbE5>LMn;hfW6RihxaXu|p*oNR8Sr{AQPARoQ zT3R42?;R#_NfP$U6?DAh6C*?A>oMd=+&74_mz|)Bv!kNY4S{K$M$K9=QjYvzkUlS> z+_1gJ-n^R_Qq6v$&yCw{+V(Bv)yu0ukPY@(BA|^cBg9-Fm_xO0YsY^ zVYMxljy=6J9xDLJ?W}t1awfOx;DJ@@FXQD`gb@_K07=?uycjZMjXxA)504QeCc|6| zB|Jq*GcMI`(9A+vlAl$BGY!PWcbZkjI~x%#v#z$1N+S7cjjwpB0x3 zv!I*KjR)kO_=(?Kmg;|?3%`rYL@;viaj1QdMk@{4cErH4api^-zHPS<$>LTO#rBX~ zL_IA?yrvY9{AcjAnzZ?gWz259^NzO1=Q3LaNjDh zy85)0kkXpxo63EHI?z3Inx0onT0W88F8b%%9g7|9<^MfT)aJR#a@~Kg0Ic{EL_xmb(=!0@HICxhZC|!01@jKjoxoySwd^n1o#cS^&mP` zrRa~@3gDFnH&hBOo-4xjTcxt(i5&m#s&`-(eAoXtVS6~CHhpYcAu-)B#c@)Yx+E3H zjZ4{F*WA96XCnHWv&(TNr4-euKZQqnb>0vvxT#{wS_A$J)r;G=_D9Ew4Y+)*5(3nr zyxDMR)CXf)*14bDnS!&GG?X2(-c-WFEC@QjFIyF-8oh$`cY+Eqi8hry+xAkm`zje^ zbRsM@*_<0CI9;`?+XcLwn6W0ry5hI>?+nn)DI&Q_1X!(#0tV`qrPiG1eKpv~Cdmw> z72nq~&=upbdcBNwq+x5y#t5+GgFPF7RYBAvDPG;7b=K5iCCiH#SiJrRAEmJ6hY0U> zF_#<}o;f#xfvm@Sm@8*3=bN^dSTyc>X}q>@V;*={W_9u)yVqx$Sisze=M#DKF$fF( zR{H@^B3(pk?z!M2;l@>K?^P#i`5as%tZkenK)TN2-KZlsQCSfFd!ie{p(Fo$(MsIRc;^eK1gn-0D1JE@k` zNJu}Q71rAuo2=avb9)>|QKJ;GCtNe1&9vmZguclo?5}_C?baSepDuQiwi7?>+g}MH zFUB5T3V4y+KFhf8BQ&{{O*H%Ofx(p4kb6&oj?HQq#h=-ahbhO1?v5t&JU2`$mhquM zg?sc(N~Lf0(FGnJL}=B)RX0u%9RLeKB9*?e)4jHj#w>uLfpox?QP9j`9y}OKN$aZ< z?EJT*I9>_RrqbtraKxh7uzFz(?r-N*DNI^$zf9Avxfmt4m+z@oQh?H_ic|Gv%c~hX)q}>07&<)f70kJYyn1s zW*hlUuPA&@x|A^#C)OQ;g+}T?)knkCkLB#BMDt}L?i3oyK|Z@$ogIVYPyz~vIlg6* z>l=8^>1e%~dy*lKRi$hw2ZsD!Eaw!&2J za~eUG5;0!pgp*XKHOR}MyxA_s@NHL3m6yp;g`YB)Ok(k2h=%-Rz*|!nEyZoLU(5~+ zFglDPR4{aJx3>o`H_MfXVyBAHQ4Cu6`gjFm3k`Vu{_{I7Su?NQSL>(SlKx9`U&XpN z^K{p|7ft@_&|lrU?ti&t*L!4glHIrSAN^S=*l^>*B)GL**!3*mwI{&C^D%o_;#YOu zW0m-ITif!Q{v-16+4GB8ubJJOEzYc|UnoronY70nFge!yqOZ~HHce0RijuAD>9Vr7 zd({L($aj5{;Y!l?RyRfR-s$9|hp+l9g5&zr$N%O6PAUu;u?bxQ^Ea!&E%at>(OqAJeFpJn4F+{G#mQGTQq+#kSm@?1EX5MP>#z6 zSQC@25=^N*)6M0?d5X+4CM5SF@FQ!qJGxjG{*o`KNbUJIAkc*&QwN^fFU#NDqo4DB zJ!IBO8K>rMqE}?gmL= z0O=ft5Re+W8Bn@mK)Qx5X$9$e=lPxUy#KSVi%)!;*|YXu_qu~K%)eYaO1YnDI=X#T zIK-BwI~^b3j3OvDsXnKq`5I5Wz%He(s0M4nIyjJ>tnZ5XJ1Er2%oJsVxqz33E z1m4epH?3HSs(c8P5w2u_u~sfzhLItiqLf z{uv`N>P!CDT7{6Wp$;Do_J)V^CZuN~43k-h9w%?>NL3JiM<- zxYwq%MA;(`$1ePo^_o9V*{izm8%K$0%q4REfwlVm8%`OR@5NNDKu*YtBnNd3J{Y?1 z!jRfVxQfV;uW!pnec*U$S4lD7yzUumMS~^x}xQ+*>m~j(3 z8Y3^+>|9(OlhVf{a}2$i;QYyDDiDKD%P%9h%4Q0SeY>vaX(NNm3g{aLRgu%e*fQTd zO?)a3K}!VCn}(R6o0zdTeJ0pm{A?vI*F`UVx`rCBjoP>(?qr49rN$WB{TUF`c{GHy z=&LP2LU0C3i8KLy)N6l`=f+;D;CV;$Y3B&3kH%4x`d_kb&c8CB-@n2aI_=?dQEt`t|uo$EjuJw|eVOam?3} z?o)GF1wLz=KS`0?7_%;RnlXaV#?fDo^ep)|1~70}TlbEAd!ZbkM6V=v)~`QxvaF1b z6^Y-bU2@YIm-ZD@D$dfxe&V&z&3+|!$LR0Vue3Y0-%>Ne;$vs>j!I>+K6TmZm(*3z zp&$^cKDY?$|E73GUZHL z)pe}Za>B-hQ#IleC>O@jJ*#nZd(K}?*>Z68UFEsuuX*kgjZj^$JTRss1d$fEavY6D zgkJ|^-u+v=Igvv_YHUq9;w;;AhHB<-_j1<<2iv|6sF7 z%A$HvsNA0mXXiymiT5hk+Ub*CWmaB(j4lGGuKQH(E^~QhHcm`i0ZZB#$&M~FYwl{! zgq%o9S_d2wLqxQV1s&6V#eTxCVjw!5%OwYWoSWUfotT&}jO@5xqi4H9Q4Ev$B_jQc zlK<*PCJ>{?KJ;;+b5H4MWWtCv`{Cc=yop=LWu-;P^{SurkFl`UmqcZbpT*YC?5cSRjUtXiEP^xj1q)(fFiz`BsUzCbqF-RvJovsW|f$}DH z(l@UQIePP;sQ*Ij6J!-sz9lFZCy0n(XMJG!pdcE~pf@|N!JpsuS7qGH&Q=1IOk2UY zO=-mmQ?5wZ1Y=WXY&%NngMT$qV*ER`Ie26tQ#uV}Gr`zTc5(oZX7BtFNoH$HCS6LJ z%-7UdtLBM#ZAW$UJ{{HA3Y8AYK@0!Lh-SCpMQn7U&CO_(rQ@<$_&xeQT@BLPacgtC z(-(iStx*CPXl>Jq`r+c%@8PZ6wKr2DrvIYfRy_MpqBz4$ zup_mV{V0_3FZ=2x+l_=6$?xyxpm-UnsYWzX-n^*(B_?LPt-3h(qWo;HMjD?DCZz zLqX3NBU<)_f{=rXjchCi+4dV;1@_r6T*wHTQg^YjiNuw$6E&<57M8BhRBQmDu&Bz* z@k*ZewK7GMt!vJgN0=$R_LYo9b2;;`QcVD=M|DK%3J7!cR+sL$xGtILs&WV@S}f~) z!CsYZh&Et|l9l)Oi3=+K)L)^nOF0xyWQEPbI9G`kDVu_kKBp~n!O6&HBZ|!ex7h_+ z%BZM_|HQOv&H?LH&b|H}mrl+er~W>PXT**i<%`x&AA~$RSvrG)-ZTBxn#5>BN7eO{ z)a1qeNr*-s^bnxTb}-HiWB#9*plK<^IT++Qfw z9?s~M`JOB{8|1l#Vv(C8S`@BwJ3G(5(TYX9W68H~ZeOM$3)qivs=51Xal4x~u{iTQ z?QRMXR{M;6ny-7$s3(OS+uVCi&-_na;I}>9Rcea6h=jrC{w+QGh&Hq7YrzjF5`7Cg zWaJ}L+HR?m48?A%zDQCey51k(t<`F1jOzqm?T(3S(4w^2-HpTyDhOOI8BEhAw8nT} zUNv1e+(q#G@D3EU(_DN64M{mRbOsEvPy|ml=HGBV{}sAvc`4?eOMNvv;~09GfRR_3 zS<`;I1$p~g@|gRUt=Aa<8U~qSn77nOG_--K3>jV}YIsgC^`ZR9IN)X@YHycaqt4h=X9b(Q6^@TIQ{)`TBIs3-qdwclkp{<8QosYx*r|a@W)ncnE zuI|{6U)c~w4tx;!*W7Fw-54TS+e{lNTJJ(G;V=LhTOw7qDl4u#pVmYPatZvy#RM`o z2XqBc%UhQi5yy5H!o5F*{3;hud5zFYRdVGn#FkTNCQixKshz8)1N@K!Y>AhW^c z&+7XV;N?!{FrIk%E^E%qlanBiGg`Z?uf!PhG%TkFWT`Clf!(}N<2B<%W>gf--d_L2 zgpA|L>>g2MT1N#8%X)>E9r4(=t0Qx5LuT|%IPWiXtQBansNf6l&fZN zm0z5dVz~>lkazLi z#Q96H8-N!t~_S3xSr?ja;#IlebeNf@} zz@E|0DF5x)v>wba2p9aAQytZUGK%D^BM`=dZ)wLg^zyie3>Aishwtw4{9?W{oVN7B zHx*AAf})mx9Yd@sM2+b+os+?t5C zAMAHP5yQE>2*0p(t=rbhO2 zFB?Rj&zG{yrk~s0O<_kh9p(>P6rul$=~wG*hLC{tmA!PwWb@^}|EXXvx{GYCw?F@( zF{z21(?4H}U*OEEHZ}?6U-b9H3_jnFyWhQvjCa%x$3WqqfCYp5VtoYs@}#m8F6$=& zw{dL8mZkOTrBXp4Gd>>eql5SZuF$MbZ^!%H#@>TT?4(+DIzpi`_fKb^shJseLWRvg zUu_&icsmd3P3Su_AF!U-x;GI3{Y7}}NE$&FIa_0e5X_i0(Mn7ZT`(H}^3PZIndhFn zOlJ0kLW!WsJYE|_wry=cIyq_ z-6Z>kRDKJ1&BAcjnFDW$eB@cVz?-|wh(t8KLJIO0c|>0nIX2a8&zf~ih}t=)h3M<5<{{3OH$<#+G~0*u&%_V#kCz3F$&&z6He{4t7?CwT+Wqe0W+Cj!5I1yI3o*L%g7jJ`8~G3Vmq zaVty|{@!SA&UBSlmYb}>o*ajN95gQOQ4Z2yzc1QAbnQ z8QD8Fkt#VQX|=Wgy@b5Fc2Y~GkpGTA66$t#LjRryu`12r{8JG+&!qhecs@9W^Eh>d z>^x@a@4KZ8FwX7k+M|g2FC5RWFZo^LkiuT3gl^2)yU}QfYgE!dHl-<=Fo#)~{p4Vj zep8#PtreV9^TX-lamDrk2X{RLv4iGsvmYf$N-t%tbat=#7O>0WCiM&_ytO(fHTz{n zWQgxNq1eA+IT6|B;jc$?S?(!2FE9qLPh^@;WP3GMS7VqkY>lTq(v-(d7)^(@H?qnS zm(wfx0h>fcwqj+3RkS`w^&1ft!QA zTM2AyriL6{(|lS;)Q4rub0?=6qbw9})_7@%@AfwOvyUpO_1l(?j0OkV;>c^`3DC$} zV>9w80Wpy-Wxc;SO|$br!(J2eZJL z!>M}mBp?w+4LN&6Tv+@mCAR8ExY4P&UYy03EOGl6Q{G2=bJeDleC2G!zR9p2#6~2i zMyc^X2J0GV92hkwt;0?F;5&WL_FJ>38>f2iPQoXh`%6MisjK9!sHgg$CxUQkQ;{E~ zqGgF#V?X=@eV(#B-27dX7XP-yuU+MuO&)|sX`&?MfE#lHfv#3t-}HZ`!v88-((&-9 zmAXI@FnR@yDFu$4+v~kNrm9ebsxWNjDZl)7#7I9ki;4LdLuvhr1tUF^d`~;hG?-WZ z4P&Y}F|n$af(q_eXbzPThtAYHh)YYR33bvC1VTLngu0}$%@hY+YZ?%!LLy-u#3t-H zy02REI`6+TkB|V14g`1+O~09t7@v7*e1ex+>$15 zHSP1f2a~bs&2K5g^h`pZZSwS~pmxoL5YuFss(18x#Xs-VCG`@&)*Cs7RmyO@5$+;%p`xqP+ z%nUNY-yic*UoUw}n3JW(!C}fg{7@plljW2Pf87D&{-_Zzo{W;JMh>hIK-O+W!_baQ z)zL=-ZdoRhH-p%jRT>N_yjr`b+}hj>djw1(Hr67{p1$+?Z1AXZ7x?@THQYJXzOkvj zv)%34dly@Hs2M;^Mi4>yCTerSCBE!5*R#e94QHJFWw<c6;r0n zIEEF0lwS!51ggVE0ivz0EF7IeVIO!_le2mnOAxH;dqC(g6-NgrLU#J^=%`u2W3X5Q zt9wKiQ5K(*ifM!f22jP7frKIMd#Tk{0LmDx_UoKg89VPzNcG{g0}HXfwyJ_SSK9vk zh}iE24F}_j`@bh(gpyd5*W7RtE+8p58$hB?vGwu+PjR z+0o(eV8QdjQNeZ9)1IfPL(X1vZO94r`puw#+EXJkV@pr(vaIC4ZRE48{7S1EY%5L) zg_0g#(6w84h%=jivACZfJg5*a-^RQea`aDU_kB#QtA_8I?^?`yykvzC2V7X@K z`B#PNia5u9a!serBJ8qSl+!;#On1j|KP3s9HgRUCr%;D3S0 z{8m%IEDb;mP-mTHRD#K59|0zcufxMy@jW}oZlTHj(8CDQvAp7BnkOsjkBb$>uw}6( z=jUXl0|L85Y-Vx97&uRUN>Nb6d2boTf9Z!yR^&9pT$=U9mRSWz^#me!N1+F+TuPmH zLS|+ZIx(-DQeiDcXQ%l0$5Dq;Y&biOt2r~4(tN(RPNZV3cS z(%TEqJqdl;l!PPe1nnrpu}@1GUXG&-SKyThdMKw6qq?(d3vPCooa8ReErx4=4n{*j z?-_u?bJo^OOz=^r94|Ue?`2;!I#Lw939v zEMPa$GV#;X3|h+PDrgXV`>w#v_5N*Tm&a^}jX0p7>CfiH^G@m92sEETDQKj5CI5C* z@Y=LVG>K({g5dc&N#i+ihK;bm>7YE}?1P_lkyvf%_N8={{g;`72A>74$=iyUO}^V$ z-ru4Dd@c=(4DCSFA*k$xH3;3RXzBZ|jE3*@#EDvfRUut}`QDDhe($FSzz7hE=_Qz* zQz}o8`6*vC%GwQwzYaT|C>}(bML^EV1P2?A>u!!81HeG5uzoxj@m0#nucGam!0m&= zzLU{W|L2~i<@P2Q@0nDs!_ol<(bqi3%e;|9I2k4<*fd4`FMCQF+9P!POHkQR`tfkc zWVr#XXz00((IliciTrUNAc8bzhcDTu70WoYQvx(NAT5CA<-J_584y5MsM0}E#jt?p zY1E|=9c$_x(|1?A2)XRlui4-Ylmvar;QLXH0Cwom4~cBClGxGq zGyry9RbPoUm`34_KN=Q6wP*LT@_XfQ$1NwPj=2H`nr?M25gPxnF2Fs_hX@VZ@g?d8 z1`(L-IvFKSqyF5Q&{LJew%KS#!@h5TcpRP!Z*J8qbs|2WOtL=T7aB)B)bd{}4cQm`wDP76D|=eJcz(z`6A!*t2}rqd2?x2h z-C@0L+o|7_HxW8|L5rd|vF9iQ8m^$5jbCXIhEQije$h@tyRa;5IUXA;&Mgyf;jXH! zN{mPrihYm4DlliLm(v(#ER^yMNWzuLz!*(rC@&9s@y&2SeI?-?aT>H8mWh#a;f(yw zN@{2kO#wDG68RI>dJu<>cHy-k&7ciSv36oz@Q4aFmIem#Ub`^@#^X#C_p7{y6KI8e zs4+}r#9+rVZCMiep{r;p{IR0UEnhT=o59#Y)!32ef+V1a*8YNUhmKxS+!f1b)BA`aWAh^?kFrcv*S`HYxE(gpy2a?}WUo zd9G;x8y0WH{?nz|JXG}eaI%-64v2jWdfou0LiYP5nXD>Srk)>pUuT?+?RnV#7@AkH zf4KNfnZ2~XBy5#!q7{IsHjHQhW8;o(Wlrhj|1o1bLrkrIOyml>KK=Q3ka~>oc4h6V zWnZGq?fucg|J1GiVzbkI($jB3Zk_}b#FO9A{h9wpS+m?u+4D_)Wq$k7PJCr;vqk92 z;n~CAo8o8~`-Xq2S?hLW7W%zrx|jdm#^KqD&zz25#sb08<=e8*d#r6jX~ZlI3Tc7M zoxg>Yzee*uw9GVUBk^a!(@YYrQ_w3r56pIx?tiMvJ@uu<`GBtg^K}wI-|5!ovG+>7_@I#tgYe^cX&{ z{@w__Wil=)Hc^q)D{_jh^JFe6eJe`l7gt2-EKw?eur}&#Rv@`G3Hb}b_>L?A3MSz( zjCU>_e5ULtJSCCQU_c$`Xf~OQQ3i^-5xgM_b~(86HGRa?N_lR4For;cD`q-;E}5hU zkH9Z|x`62?3WpszW|pKINpdAc1{MZ6$4M75=?tcm)!ZP!YLTACPy}_=2QF!eK~}|S zG0cQV(Rx1S7pg)0#71d&g%rmRS9O%j>ce8WEBo;G{^;v)7z2|74Z5pz<$Ph(1}aIQ;b{uH?O{Ui-$C%a;|q;`&lq(>!p|m-w;_?Kt|P?KC_C( zQX|Z&iO*44Eck;biXd=){NFtnX3sK;pU1cdM14LW-jqiKo^A_s+>$e3brZ16bc&Y*m6 zQ589CSuFJnsw2sJGhDQfN`yskm{2iT8G%^D@gFQV8D4qv5@vEILB9~`s6Z9R-MjPR zpJP6NvOCn0bD*5m5|GA9R^bcbAbt1}iXTLcb!M)4BwVf~P&WtDR>KuKj={#$u68Ap zo7{t9>4=JV1k7Jk93n68~$^!ZguX`4%LFk)=;&|LX9=Vt^eb_y)UdWuBb$Ex0D5=TiR zVBT=X(BvO4bHQZ4pK@(#D;f%>5r-~Qa>kv54;J>g>CoLF7cby)$Luj=r>&u-+VT5;TBhLi*W0odTX=_xn?~zN6w@TJg_^E^a}lZX?ovU(ga6D=|;aDbYAq z7%wyzJH<0G>gGu5w)0V*BI>;i!y$?*zM?dWg)J^CtoY>76)u&WrOwFM^kGL5Yk_!1 zeI)5F&ggiJs@Ro-GpZcOl3TAm>WzE3aBd;^6gn5YklAo$!=dlk5tEaO zpBMH9)o5aKQ6d?AiBUc*!;b1UMU*Cz#$-p{$6XNRkN@i4d}HI*LBTDa5q!#oD~VPUH2%amGj*=JgtcHB4l7!O~+~b`6!oL=j~qn+s-y=w*S66)&=xl6QpLnPq;z9iE)A&kC z$$Yjf=-)G$-`XVUx_C3m7V2{`h$Xf1(AW3S?l|>ic3(2I;+N3-XZ^XLc0uB@qV^WP zj+inzrzeXbKowM~IdTve_u*?Q6?)xI`Na$$?%l$Nx8i&H8jt0pzFy>l&u+zw?%n|f zCK@xmes#g>LsUaNY@{6=m}rY}=Q3V@3Bn1l>7c7WM-B(gew zzO=nYsqclfgzBMDO7B(PN{fED;fSf68H;+=A+XoEw9$yv+D>D&T?^zg&u8eq2c@HA zuw-Jh28O4R&naf$K*2JA$_fp8!OY|p-xmHg3GYvVJD{3s8!ri6EQvHOSbK*Y&6c|- zZhyRw;cK9(P;B-WPClhrp${k^j7*05Eu!~3c4qMkqLg0xJS+kVeW>-YMyh}jX`jbV z;GU~OsFsjHV^wBjX%zr$qCushzS06Pf;i7tt$SIvdx8=Zv0QxN=H86a)DcKC=45vU zp_F7e&gv{FaZhz+I)K$}yd;-FmO+pih(<;F>YAvrJ(GYTKPjt2oRq3(Tn=wfA(PR! z?UhgYJ6=DuWLUM#4ynKeNxh+5ijF1%wUsl#iO z`>PZ2qpC*N_JdNV+Mg|5M%VwlLV37H`nXuM|Byg)zcv`8zf0)QeH0JHQ(f7bF>l$A zV>?~+BnahLz2b7}3K;uG`i#idIa~L<3!-b>*4g)z^-w9FDu)Do=Esk1nhQUl{#ld4A9Oo2Yq$?r6}ypDgwW zYHUoI;82a~U`X?|Wg#9n2k0*1FI3*Fp6R6$t1!6jFaZVu`i&qmxed`oM^JM^<5Vbl zEJLCmdo5y)48w~wS=@7i1)ip@!qH)ruO_0eKZvD&KB)M~i%+CHhXWQ#`Nln*^cx-> ze(yZ-0FjB@=;#I|&HFh(7{i+scK7lhZ5G+FUjU%~d;RG)^l`6sA{eKjU{;Bb6{|>i zUjrCyr#Dvxt$C>m)4Ry^f_RC87yX4Lky(XjQqIl<2%hGp2GVgJY;jnm<=`9J(>;>N zl72OXXBh*{W4wmb5cr*~R%k8`P*F`|7xyZc6rDgoPZn^&aIMYx zbz&?s*OQ22k516$^r|nOe+U^w+eFL1;0}r^T8jJ7ij7&|=2G8*8{dcc3K~Zku6{+} z51yWQLtSL|m5a{v7$YHRH20YZT+)=`7xo1{Jxlfl8;*ZJy)!m9H@C0|ovSpzwCs{J z^B8!AB-1+me=Cv;m(BL08mawU!Ps)}@9M6Ov*AuGnXhgCmR;6^wjGc-<>l|*L#&~K zK;)(1dUI1y31~1n3z)niHvQQal2Cmtx5$TGGZ?4)|SMw|J?S0BDzmWYO9EcF#7Z5)F*GG?I_~&{R!bAbsALQ!Y<2fe!ckr zKV9(TdQV>=-J5QUr--^&EoeiJ{kmrE*X8~1#wObC?tSwD4>qp67u>3l24BL@`(2cc zXmyu@_9qXE%o-R%Y>JP9KOn&aBsM1W)6vEN6js(t;cjM!js)EV_c`JU_P z*rb=&oQ58glhJpwsR#ZV$x&aHORL!Lo>a2Cpz;?#jXK57Z=6(5;g^Sw6*&--MQ`)t z)8*o*9{+52-o9kNeEuA0366a{o3R?Uk*M-U&|ZX4-{rhJ5?lsb_t>kgF`hOAXva*kQe{Uy4@0jTWh{Iayr9=14;pF#Vg@BI55ymA-BCVao}y*x%x$;U=rguIp4P1521@gezh_QU(PzikCR@L)R! z2vAXvBQ3-0pkfFAbSPSR+$MxQXFjcP3?AQDY;iRFv!& zH8rK<=1!-$a@=zV`Z9@7@XclvW_oG>>ng_N_Wit zbY;*cc+)3+Q%1Jj1{-;3WWSO`;;IU5KvRj3GZ(-9D?jH+5!{r^#)A zVer0=qn`D|4hKn?#D)a0nV`xT`LPjs>d#FyY0rvy-PDuYrKhJ(MyjGjjwKo#_(p?! z^1k4c8_p)EnpQgf_}d`u%6`UL^0VxFRh+#2nu}S6Y=O;BVeZtG9ZN5)l*uD-Jaf#x zo1mBFR8@bjCtA8;Ha2u7LEYsn!3<{N;R+MiVZYz?(K8F&gmtvI9i5T=_u(%|X#V*R zR>2dq^!a0aJTpq@y**dK?M2_ZGGPC&bkjZmq}1+%Q>L3{Q0Uq3n6=I)@pTz|qq{AK z{jOVw%h>07X1H) z!-ddCkI4|%&^04J|CrEPONRK|AibdN!}8=l}fK_@qZO!Vcb*dKY9#CCLl z<#sBK^tw+V=?N#rBO=#kGuet!Mjy3^&C&fm0Uq^g`H-#MrmVY3=k`6 zCo-nO0WE#Jp)~b<9FY&RpZ5V18amZC-qr5_L5~U!-jZRpV>_3{hSGZ&V7J86DDek)8a!@Q`&F6BS=%-DY7i;)xaJg0}5DdPGQ1aVsBbSb^9-)dHuk*`caV zRKJ~{*WWXA`a~>PI_@|$Z^3?@jr6Rc?AT&NsnAr)2>fwn`D{ZYjgAePgmSy?Oe{T0 zOjd*Lm4m7{;t`qEb3c8(Mk0SPr#xGafOz7HK+x1U?cts)ogt`Ay<%?MwQ=NY>Vd5Q zAQ(Amxp@9b9?y{>5M*KB(2Xwmm@B0Ll$Ccc5;g9YrQs%s_9>UetA}FAQOy!)5M<6Ip1i~f)PO31e&$tt*A1Zq}LYCB2c|66( zAZbg5%+x{GCC$yBp!qcUk`Qn^KMn|y3!23nSBS(>TU{gV8IP!gQ?w_&dbPfAYwb9& z^{#dS6*b2TTdypgtPpo%Hber|Jol~R>V_^q9!xw1^y!qdp(J-4N%7hhy1N;Ru8RVx#;3g7S%dYuqrcBq*S7|4MmSi~ zb+`2v#*de7G&;u4nHqQ7Tz()QLMIZ>1hb%T6VZrc2HR4!q#fPd4q<240;U zZiXTEX9VjH_-lo)2|^#TLT>@J!FJMWPejDtijm=N6XT7O%;JFi^F$Y`vAP~--56kq zaXdC-bF5mmi*gYz)vp!+L7RFk;3J6o2qNxV;1~uljQX;Y8})^z-+2#{b`pqLW+dn0 znpiWbpHxFH`m3fU7K*~Bvw$Js$GW9WfgWas;X&R;AZz$~lpQZdkJbjeFix+$^5X(; zkK8MOXttqBcG+<;E(L$Y*kVQ+_%-S{9xGE6%2t3a&Km}8RcHpN!wJH}VgN~CYXmB~ zyR0ltH6M&_!hd2-AHH^Qay@-nnk(e~36dGn)&Z3r{sEC89N+n`T>>+~vVZNUD*lP~&`WaY+*ovG7fm8%vELNmn6LJk8Lh ze5KE46DRv)1}n9Xt#gp)g`L{Y`W)uuGQ`u@xphgUv|x1HNxv}Mnzi#Nirm11%ayDa zKmEsb^XIc^q1ueokwO!CY7AZFRM5Hp4n;OeWS|JMVOtz zRp!$Bd~MRzVImrM)6{p4NJE_J4N7|#6MW>`d-(l8T%jvQbWS*EB+Ow&12_{N z1q^9i{m+6JQB~7?bmY;QiJzTDff|9QhLlm#d?$Dh5ymPS5c_&q!c$DyYsrW50}B0U zpl!g)ABoSz7_k`Vik8e@B`Uv&zXzk%6(jFrvc_Q`+0#(9&Yvj7WMJ`HFe-KthR*cpm@FwOz5L^g(`dkuSBYB zjVz+tZ=Zq?w1?OVe^CWZCi-k_Wv9`{+HpZIT@KZ^=Z@Y?fquaUYpJHN!oSS?olPzPzW@5Y~pax zqHP0-yeyeY+-;87TZG&+vSUyh{VQa`<=?Hv& zH~al_;i&Nm@(jHDdP|vEYzV2s7E;^UV?QfijqJxeFT%)92ZtPCTv-{olP-W~IpG|7)z_1cW44Nk1THBBzo1 z?bqm}&f_f?le^EC|Lx29F=$Bt$q{6nz3$q?$kQG=*3*?7R?;<_U8$0k$I!0kT42!f zy@^xk%^I9Ahq`m=In!Lv;(Y7sMd&R@n+V|sRb|Y1h;)nn{z0DJGdhGG7O?$T;MR#? zy3Cq+v@L}Xl@&1KL3@`rdJKFWa)vi&1D+`5EEL|?FbY0%Jl z-y34kh$^CE;jBW*Q!1uSv$PW!xmnp*-uX^ zZOL1z8aX~^F^wxsPj+!`GK{J zYEWV$_EDro0ma{ZK+W7S8q~cPShb6VHwIR1svcGx-*qaO%*aCZoqz}2-4!xf1NE-iSwc@ZZeRya8(A`KMG>^7^ zwqG}5R&X;TefO)n;C86CwyO0<-N{ZP?~kpiq@Z)1nxCJLPB|_A?e6PZNKW&9q9Sp( zx8V9^3B2p!@p0&K<^=Z1`zd3rlIw(JlJ;}()zSiez>&cW?KPW(kHe2YWm)D=b0P~d zr)~GB5@stJHMN1GiA8&AE?0wd>o@TSY_5|*n=)?eUF%hgo8ms}#k^ggMPIujcReZS zG5LYp3DUQjtzR>m}*A z5C|GH_gFH#Pt}rqT`^nuT5YANjgJXOC4F31kaTihWd5r9UFFfqvKDR93z{#eW3D5} zmEJ`h5d;9P$EdKG7OspHL%g1DvkU(c`yq20^lKnH3P)OP%Uc*zxgUm zu1I|~+l!5C*aZTAmL~|O*lCBKbrd6ls@-3o%?;2mZjet^PAz$+%g6eXd)&E{iHShS zCfSI0Zg^J`JRI-*&LCnu4@b7DhFyFI1GBF>^8<9|FOcP-9|@beTq_f9L&9nhcNP|Z zZJuwhyB3J{ zzDKpDD;)*@yW|b*^g3Q{{^=qjvM_D&JQaT3b$gU{>DC38kJ6>>_L-F2UEa?l;asM; zex7rnN2Z3aT_#X_osQ{H)JUHEGU~dcE(k`MoP#BqcJ4WJxtVDJ2_aHXb{Lg&h{l-lMyc?l}{8x7X9B zbhb3UNuY4`G_1`Fg3g9BiRQN=bMEX;OZbJf8d z=*XzBA*Up>Px8Yb>X)@XWhmzy#_7Mnqd>%}9==?SB5$MqcK%%OQzWUe3Oeyr&^#}- zk9ZyEx>pY-!+;pgf&>7T@8Zy7ZT{I8d4N(Vi{0L7V7`>exm!w>nTA+A%3!?=s;a*ON8YqS;WdJ=Rh<>Pu8a%eO;b8DTt zxD{5}vpVv{2+38c$VRvd+Oi6>ZP{>k4wi8Y6J0^qfc6Rz7Z(el!dvcrr2Pi&3qpGdYo4d~ zpSCu+PCw4ngKN7^IP9O6N>=s@Uj5Wog5R{>^y*13r-wknYOHA^wgMetW6}+qI-#L` z1ZNyQst=P8btWD(ph2{NYX)X_;87_ zh#4_A)HE=NaMAk@KNVFd&Xga9K@&0MtfGxgMCvPSLVvj~2?ssG;d?yqXi^m~7G;^K zHfDVq#feRAz7or0YO0Ww{jEtiMaLEtoaMF~V|W$CW;ST%Z@Q6?t|`ZzO3pb-Vs<7H zBtW0A?IfzOMud~n@dC&yEF=<16}j;(JvKLr_AHTy0UMd_P%$>&<6<-{MfdH{aV%Un z>4o1M_V>?qZ}3qGyqK;1!oq8c#^F4Y`UdtdtYra4lFvxf{=Y|0H&4}j-=2TA=3VA$ zz*FQ3?)Kt8*Hv7V{`EbYEBOBr^%V|LaLw0rgER<8tTc#pcXv0^ut+UklG5F=v~;($ zgmf$24U5!L()I28zR&af{)Kz*%$$4X%$fVD0}34k@#h-1UtbOxcsHhd;AQo?ZkcS{ z?NmMPv|gX2GM*huuYvki3A`rvVQ;#(z0@}JBwH$god10;C-~hjQAVC^Sz6cLbv!9H zJ&>&kVe{LAuKjvBVaK|*%_kYmjGV5EeN!s`Gd2(HcN!F5KD+dmkkoF{WjCE6ma6(r zZT#}tHIK2II(RU>UtPwmjXS%qRdqbjz6or+{%%;@gWP6+oFAY_#Ne zHMu}_*!OLNxAQw=C?k7eMSHtJvDKP$obfC#9#H``un9c#NpruQD^;SawdXlU;qlM9-lswkzN)(6>B+{jMo1>4rrIya$2 z5oc3LpZVm9of&}~@R>+@)jAlH1)PR%d=gBek{lh+f> z?x&u7pL0)-vw2FnyqC52YJt$IP1ixMUG0G8Nvr=ppLV&|n#GmDvvz3uafFcF&nVls zY=4sX2RUr211?!`QW1&>91pww9Dlq}wFK-ng^WX=z&et>Wy*QpS6;{WHMifhvxB7K zjUWCTocZ=;8sGGFc=p>oWEju&?Ae@R-$xvP+$R2VQ2GDvq2Tt>--v#U`NlQ-8B-K; z-miDo(kRo9!VR%@@JbrkWCe8Y{Z>BFXWV=yOSox?mE^?6P$(5ikm?(rmidVHkvzh zpn!4AgpomV`B+8yEh^#9AOK-eDN!^NZFtmZPUcUu0z$~d1U1KY$J z93%CjwwKw*Z74*Uc7nq$gKA<9on<1~9e^6t;uuwty^65LN@Vu)UHw#xlo=je^25|S zByc;Xrn<`mRoN&LfL?R)l&tEB7kLOTxVWK$tvPGdajiggAe9nc*&!DJiA%{f6_?2j zIK_sa_Y|FxcfUh&{w(n7t?WN3_X4>@fphMqdfqiiHyxWz$l<>@er*tF zxpmjMpKIK4>FIfGNHED2eL6Z9$~U~b9DdAiH8#u&$PfQnxALlUP%F^7^+`D(5Z@hG zMERUZaI@lcW{4WcGZ$2-O+fdJIGdVt(-e(npd7`kw47|NIqI|?J_%1vS(&dA-?6_V zd#S>8gLg51^?TAozli{i_}dx&gl4iy9+mT0-zK{GC?&?VPdts}6pBHbSt}ahBqw3& zKVdGZKPTgUutpd1;f-mIF%zam1PALgTVtS2>_j@A9Hq%qnBk`2;)}({tsU zhmEO%JN0vl3rO4BF*RoFlFckZ)UX{Pa6q>4#v1!D*xK+)yTrc9a`Fhyq%9C|MZgvf zD`=Dg0Z6Q+Vmr2nrcKJ7rf)7B&1D_VZuW7T2&=IZI*EvpGn*KVPIa*T`id#u|TC???4Spz(ZXAMn7O| zK-+suVVv^r_x|DfOa_YSYeYL{kDh66AP~xndV?eUdVWRKAWV|7`HGZ_nBcwj^9{cP zxBj1m2kH2)7<$tb9;;kLKpH@j{vr;0QVYh7#%g&Hmgb@g@|?%+M*e1pV`Di=je#*q z(HDhf0)#NDP3N%8q4H6b&94%y z1$*j>)0oz8SZP@c%pm1MUon`s)ZaO4w@wuzxUnrwX1Aysa1t(MmSe5Y;U{Hf+O1RZ zUQeurt8g8I7Q*feZQ-gc5w#FVRdL+PrNC>ayf!gDB*C-usx)afKFI}W!1z=-sZ@Zi zCK!h!MF{Wf5;)m}c7k=LMTOk*CoL^0iJJyVDVBw2u_JlU>aCSSVQTXPk_xtk)rouq zAGNC~q05?f)aYbq{|SR;M++bt+f50=-mq73NAoh&9WG`_yozj1$vZbNW%pz;LJ%4Vo+ZyttYBm3U zz4<%)-84Z@!wY#)oDhidWUAip_VR8Rdb8)Ypsb7v*VJ_Limrfd1kw{Y;b#J{1Isc-L2yH6q!gB#5P&jmiJjfgc+n$zM{iXO65s^ z?Zt*>$80`%_nFcfOBrywBp3`&ZRox@M~Mu$(%udIGaWTn$*B{gg5mbH(`WHvfI6%w z(ayCH{O7u|5lw;z0~I2!DbH9^DQSSG#K?>zgS4cwMX@xeSl^$z?F?|?Dbrt!%gzW3 zO;!M7qW4YX>Y$q>gcsf4|4gsJB~Uc*j|=^r*9_J+djB1 zp(|5ENG&x8r@7k=LQBZOid`n`WcK1f9N-nLkwD0I$NAV~Tq$QSJWnsO4B3iGc7_yn z#SINEulMW%9c{3YIlPfBrKh(M5K=fkdmR=gqNGeLHddf)PI)%?r&K>MVz`|FIq?pnczM*Z$vd^5zLmo(EJ z&%Xn^#ATI7_8;5#fN;H(WqJ3#3UIt_7e=OSC%$5)tv-l4k?<(q~6H9p)c{OC8AizoaSBqfSO7 zoq|jPl9rXG*3{9lra4xu8i4=oy3LMBVCHCFDpAZBO8e9yGS(jGuy;KN;e+nylF}<}O=}SJ1cXbI*vF(Z#h{gt#QvV? z6ONnM3rCsA?WvP~2lGMIEAv2soA%+K__B17g4;Je4%ai1wuoxGCTu799!#BNfaYFkz;>l0P#78Ow*l@Go zDBo&m@nK0f^H;AXrNtv?BNToJVXRO3`AH|rKnpdwf7aI@qMkYpfSZ>$_KYZy0pTK@hsSJ_DEbWb+69AuS)gpa$XNs^Z7rk@iY zoL#oz5-x-k)PdpVS_k$>+n}}p(tB|s^z$5w@3FB;YFD}PA<67MFdqqyuak;1k6J*X zDbiH*7MwgZyslxIHnED0$_KUEr-c6IynFzdRhN`X7@Z7{kv6`9wbf`1A}Wz`El`!| z2`mvrAi&PSb1zi)I4k#}svC0_X|W$L9^?oIdcJ>vcYfs2F2Y`Ppxj(*1Mylow7;$C zu`{-NuiyTw<@OmeePGgZUbQLWwLkoF;*#NaQDfTnM~8uR_s7A%SIkkG9?sBZF*(;* zFAD1LyQ^t+5OLb-H#ME4*{V>Y^gY!(u;Jso?QJqE9DaTYfW`lrohJBDH}_t{e^Lmz z@tSh8;un2}A_DDt{?^*#u$i%b!*@6FtoytDqJtb`ho2%ALR^`Q5llEJEwJ6M;ZXE* zTAyZc&5UwFlXzj<7UjWh&G_R|4{0T&HOIABKo zITolJUUV;3?uKtj5hzbogOJ*oOSmb5v?fy`vjxaGsxl)#$4X=I6VQ3Z|D2D-QiTL{&`jFm&8wV3 zRINsd269fs?3i~|-Qhr#lAl)=CiQ8vIJ`JySgT9L7dLp=?9XNjMmzTQQ-@E^WsKsz z_V;Yts>cxx9neD1J1Z)*)9ek!1majdjNiGRgqZ%VZG~7|zNU{NstH#QU}LMBUPPTu z37>MXfDTkK#UB3hL58UQ^8FedT#WSuuD7_`j}HdAgD~O7<9%TY->2_d@heV^%jdgw zCVyYNx?ZcY;U-KTvW9!k6VC$gDtr&@8vd#4kHn8u9R7Q@nFtsV(GUV8 zX#p|@#MaB%ImwM%xV;mA(cVHF+~M-{zc9y+CEhsRxcDR3gah{Do}dWN{IL#AYGozl zNLsn&cYb@o`U90IS1U;5&2iu4ciouuWXrj2_U5@`X=m4Bs2de|47V(?XOq>cL9DE& zc(8sJXAxm(P79^1x_-1d^gE$f8!Dcg7Z={Pm2KgkD9p7dr=seE8t9$cv({)CBD;Jn zG3GctnZ3HDZ7Dfg=LhI8X#Dj;I}8NMV{T0*l-`5xZZIn*zv=kmre>>9I>c&|dzHKx za3W@#e!IEnwx zkyFkq$*;h^Q3eYCr+%v26TR0nrMm9FJj11^OHb2ce$RTqt8|w+vW~j9+f@-*LBd!+rkY-@ni^X*ab#k8?Nw z2xGB_{ohv!82ny_qD>r!Nl;*AsH&*y>JW|f7J3j2%v<(s@?DkMMi(;r>L{ARW}F=j z-JB*hsa4_DCd_3)4vL^sNG+5a5NUhlA?fZBoK*^=Pc`R@Db?*BQA*k=5n0uFMCfGD z@G+8bB{fp54HPxx(P#nI@btJu*62G{7Y(FBVT0U5KyFg*5D6iQSmK8U+tN~cDKyqT zz%+aHu?CFlJ03P_ovbnHxj$HRHEybCugvZfPfY!L{Ea&v32s2TZ%u*MZjaJ3H5EZ6 z6*x5aeO7|eCuImQ)^-SU0E?04kNYg}HEeyo5F^I&J@;4j-MEM9gUop53h9P@;i>g&c19#^|YgMwLjK$T-z71TxNEW zQ^0;Tha4H**ITXWpCIdPxAn(zwemG)q>p>j(=03ir^57&$`;4jKCO`E+b>;$b~-vZ zz>rQw4R*!_#UtA2ZAuEF2VCLBxd6&m4(x8VzRf8 zm3ULAS)bYx$F3RB&ZiUb+r5`5fZJN<)^2Spz}50-Eq*GVuw;^4{a%}OBS+ioUzkM= zArw@R@UCt`Qnx3zdBM2bSN995!<`y6xl98J$VeW zg*)=pxoTS6eAswbuDl*NGPq~(yIqQ2aYfJ&oWttd#T<(8)%G`ol!#q{ED~(e7ekrX zPQTBLpIM57Y++D)enEwcEOjcDaJ-M!d6!#j0G3~a-Y35`8(o;jma@ec1uu6gxoYrf zVfH!3f*I-4KcW-pFw!F7p?6grMpaa(r{mM&_pZqz%c)0dXtl9ts;Q~+kQwvDABAN) zp7wpCX3~%RJ>*}jNU-|tN-ckGngW!`^YYP7PQ^QFkKNJBzanzs08U=M{@lV3nt zCDe!?PRAO^Lr2dK78n|lF3k#0VjQFonTtb%ut_6va-skHBxF9=Nl1?K6Wwk|dJhD} zN6@y61uERQ0#Sq`+l($DZJk5lPkejQ(i)S*2T>*LtY zUz44`->K_)s~V$zvAO%{O=!Oisxm#-bApAK9OU?(&GhwA`1$pd>-Zm+z`H1HxTE}l zn;(mTHyP01Lfi8PVh5K^rZ*m_-t+KPpp;$fQRr+hFVU1o`9*87m5*llZi7fSJncYa>qpt?%l98dw=>7?8B{;(dnT(h^RgJL z6a}2$7BjB!8j`==0|D@(H6=BOOXS&`Di?RD3#vz$7C7qJ-4fw~5+CA`%;-Il;ppPn zBRbdVKa_#V`IN5r4_`~^0eud8`+sT2)KjhTVCk7cl2HlgYiMp{=5~;eb_|Ezj5?ev zUmVDeV*&FS1|J(cRHKs!cpM#MRk$2>oaaDBVAEFfo2=tUv7WpZhnW5Vha+BIjuxhw zHbxUDISsD*J7rrK5um&kkr+B~?n0K|=Ac)OALMEv^D{sE2N) z1HOV%Jc3m3C2T|y%D(vQaF@_YpHFpI)=Imd9F}$O$B@|Y96(!vDIx`+n1v8%Y{e4B*@F^N2>KBlqVB3RkOLW z_dNB_eL6X>xZUO*Prh6|F+jR{5HpO2lvI!2bzPSZc|y2f^6crU9(^C*xt?&k8By_d z`CXB3@;xTm?+T9Js3~EcACZ7anzh(J6bpJ&#vmq zJw9}q0tga)!EYAq&qt-WN!&jX%MG^@5xg^zQM_6ke*ab=Hx@`0z!Qipk88k?$-3As z{No;4$Vjiq6r)ZYrf!ff5h{%}wJ{$YgGkM_TN{rqucZ`(8}1>}=~2alt>iq*J#0Pw zJ#JERGbcQ|b5oT zT&(f^fN1pO#*79UTMb_PdaP0tJIgJC2`pZ87C|{>*1k0YCnKm(uEC(1*R925@|G{Z zi*T|xpxL^VH{W#HER?)JP<^r~;<8M7{dZ!z!nyEn=IvTReTMBkn}uC1&`xuRD+%$3!y z0VxR=PP!F9=utIwg2f#3@=eT;R5iSBGBc$A6Lv7SpR}=Vt&kwHd4nm_wY3|`fD-$v zszFfML>4n?0iVEVpN>v=A>&xGG_?cBv1)9f1TV3&v!VQRTT{J^V|f3|Nj3kPN{xqe zY#fiF+?7PpRY;h}aW-}069BZ*eetJwrl9 z{eZR_RMm3XrlHNQ;X6gGbPi{L@&~#U$J?QNO@7s*euU&{Mr-uU;)!o;ZDgqLI&Z?x zd=8+f(EPgXcPO_v!aKzxfdUO1yn&~tJymeH@qsG}RV}YGbBL+}pQc{bklEdyA!NR~ zt*zx#PD||ad>PPY0BX7QKS=f%uRu=3^6UXV`!(Py!s~8_#rv@D&6s1hXY`Lozt+o8 zE-&9D1K(N!yQp@5hnJ>6A>W^0)>D5S8t?o$Ec(d56O5`N!ss;Uza_H9CKfyzRl!N9 zZWED>=VOqnqoa`3q#%jjOJT@Gm)z*E9&20wo8eXygKW_S)q*pzsdI3tvZ5>$++dO@ z-|F*(zP(rPLT==p^9h~ylWrIqDtZB{q=t4EhICD5=Ww~BHV+S%g+gO|Cp^f`aTbPJ zL)v{jBaGZ@Rs~@tmbB&S`Q8YW*$xeF^FVQ#`yyoVuJD3%w8Qdp>M)sXtiX;*bZA0L zM(}H-mNB7+i0&?F)?Z^L=Ew>qS=AkNjIjn6@5JnJi&_3ePo&)sn)6F73cE)qTuAgN zTQE}{J*v(Ci&!X|E&aGZg~!P`+f-(dcA%<_9`-5$p?vT4;f?DscsS*p%Ra8#=aL+Xja|Me!)%brV~LZ~%&MvR>aD1k zN=NhM$a}p5xZHnPVv%vLx_B|SYt)06=kT@wxt{ytqPwrhAFrmxz3$AZJf9jFe(b@# zP(~%};|&qaWDi~(x33H)+hZ6aU)^8Do*^!q6LzY^?oifLfV239Y+OV2m?i>26Iw%J zWM&28z>tIT{+yMiETEPb4+EZ~shz7WSsW#zBVn~#%bx(=Sk|=+^W0NYo}^zWod(X; z9F3?89~6cJ#3_(>VNaJvR@_%LT#OUcr}fcXGY2_39B(_vmQ5hae_3??Np9Cg6;I#tIPcVCuIk1#1tCZQM1WU-7g zg{Ag?(PIUDrts|3TybYWc@ivBhQUyQh{<%6Z7So#Kh=^54oEKwr8l&Aw*RexMEo5VC&N zk9~{ZR!v`8#!y;{k&f^Qzq7O5-~d8x>Bg)2m49M4ipq?Fdl$NgOYCsq_%=I;zvN8( zv&9$YR62MagHm=NK=A=L5}Pnu@xvjh@e(E+U|^}mQHT^P{JGn}%6xul!Gj_Wg^TOi zi0@Z*r^W%~$Jjip8{{QI?G-|G61c#!_#Z;{kSqTr@bII}%Z2IVE61f^{%@htrd#iuH)>DY zqvL8*MlPE>CW!@F=^87Z{BT+{lF+Y_QIcrUb4%|Gbr?AD=-Fb_>*s2S<(o7WVfR}K zj8u;K%4N2#%VVHQESI=tM(^tC9m`=W7&}*{qH0D@sHv(JPAwsss7|PJ`Mo@=Pij<_ znpllMuA7lq13%otAtEHeI6?b;LA2wkMh&RdaY_C;NtyxCA~tkO@<FBB~gdbK&7twBR_~h;y1lQ!7;QB%=EilDH>!f-3uRL(rmHYC?be^Q6h*DWHKc zV5wGs-*~p|{k2Vd(J6hspoj80yC^N|5vKwi0+H;2_46{=uFQa?kGrwJqGBIY;1|j0jo9kAQ!<+*uzay@^`#CN(fx{6VU$&C= z?7H@yMm3Z3?yHyGgAZ`$8g&ngx5DNLt8UU4W3!u2d^UlQL#lPG=bx);#=e8&rq44? zYQ~+f+FdV|<6=YAm zl7VGU937pwEJqGmZDfi)>Z?M=Hb!&k9yO>rEEE?5j2^Y<$R&o0ERdrf8qA(|ez;Kv zu%w}%7rHmUxhnQ4fjT^5f-N8+Ia;`T`!5X#I~MOR|USc8ZsgIX%241&N?1>dy4 zS>r(FEI>G@?L$j*|LK>kE^sGTNRzL3V7on!t!MvHg@TTleB>qx)3q6Nj`PBOFh-eZ zV`bmtNPI|3?G&PwxdCSz=%m}pPNYh+pu(B!a`C;#opuZ@J`uJ^A?4b$ zv{$&)6Mbu0oJpNjzC%nsuNh8;9d_2;pSIo$Qw6zKLCt^PmiMHyl;aOCg%M$A4T z=6bWvp?3N0asGzyU78R#&OKjV<8Wb?+x?c&>&EL735w}>n6B4$Y&HLLNJOA>zFj`- zpQhpYJd)O|VOa$%K!aj%#Z-o?6Vgr;1ffZ?t81i7;>*Qh8ZU^^mPLxHgvl@PaM~#kttPgN!u-lW*t%*A-4VpwbHgt*M^^zykMfXm2 zIzCIAZC~7u5jBo0Dw0;k*k#O>Qpr|cN{4B4IEpb9xhW`lXK!H7xxQbevn(f%aE1?G zT}AoF*6Bg_C1POyd`_;%^GDOiJ2@&r|HqzR34u34-RCLen_q?kKMtN>Mm#Ma1fE~K zUbpcKp8k=L6Me2Rz1trA7DU@+_Mlb?Fn-(Xz2)Ol&T&A<_BcJWcLb8QpCR;<*1v^m-du+5py zMhKBGR_9zR!4k%6!$qB|X2hUcEGMBxObVe>KW{vWLc_~sYAPT4qMaaH1he32@o+S^ zZephZWhI##C011kMheVpG<&GffIG9bsBtGIDPougrPrd~{V8-#%rj!HwE)V`a>D?Q ze2!sPRC0JBFeW;e-Du7wCjFkjmQbeE<2ZTmfDw?SRen+^a=@7enBEcXvKi9|y z6#yQQ4$&lfUx72VY+5CTCYwWcj2mXZ;}sSsitsgy55}``HQk~M92Q;(MbOa5fHPl2 zWH~vWLF_0(5lB**$-a~@lQp_>3s;;~VuAwZ6oiL3*HH__4VKrhw$gO`VCRrkT5B15 zD6v}(YMpGp@`qn6<}gH)a3XNjBqE`3R^p|2k1uHhIxLKr={PD^-KSBdB*u%IHV0DI zcg93kpeJP!$^60h(A01Ah|@OC&+vlK2}h;Q=T0l&bz5!pG4mg}j_KZ?S>3^khZHW;6(&2pkB>EUWr0sS zqlQ)vwx;)g4)Cf?0OdOJQUY1Wg4I9(1>X}gVfr_T))OxJ=RL}`CU zrHD`acr2~M#h;rvacavrSv0b;AVeW~*{!CBjyF2Y)8IPmz1z=2>0E=?nkQYAUHuHS zMpLJn_^{+>Iwwmw8k+;NE+3v0&ovH1tFcA5zyq`iZec0Low8b#A{A#=P*3De1(O=# z)7K-)Xp|e0{KCTJNV#hxTho@IRglLYm`7*7E3`^(QI1@XmD!4PHn&|RUWX=S;nN@& z;|*-v5@JGQa`$s2U1x?cGKCtzoh%ec0DO8^9^8?XOs*On!0q}X#W`x=ZASYT^M7>M z|Dg-ohxHqUpN3{{9-x0Jea_BeuhmXO_`NSelv+<11#Fg&TsQkm>ztedUQc@bthTxC zu{Pe`)ZK{vN8HrDljOtlGF*Sm<<1RD1za53tEF$_!!>k@I^L&RGqOYm$ z27-QS-Sg1ErY(`BC#pw3Nbc=Qr#FoJhW%ggB;_?eTJoup4A>t*a0*;Ut>%waR8$DK zq$#8&%>boV2}$bwcu_(8)&p(!Unt@n9Sqx&40Lp@HLz?BBnf0@wDH?Jvpf%Z{wVW} z7WenXb0_XGf%S`pgjAUx!q=)X*>#aI;oLCsHFuPwcj0k)q9e0B)=kzXHR+JY&6%KO zm~Dr=7^9z?_{$*9UWl>fRrrW&%EKOpaGDP|Tk-%^yC=g>?4!~^sH(H#SaN$jwhBj0 zimGh1mFTsBP0#&*ST~0MJI#X)0?sM#m+ogNz906hsGc)Y!nO~wRL0%c3)ylo{pl`M zYZfw9eh+PxfLF53=|weF-d%q+=fRwF6)WE8H4 zc|ZN%h>h=4@}u3hD-mI%X1fp0UJ*hZ{p%B8q1%WA^15WSB$V6^G}5!NsdnbE!_kdB zZL~19ER{6_er%%#G#BS$W+<&|Tc!&x&n#(;V~S1LiB6Y^8tMnwU?7-!VmdVno7y6V zgGapN4+TnD5*OD2_p>(pM3GUl589rcG8TaR7e`cNyTICz!C1~3r}%2iCnOKVj_;Q4 z*oKEVs?y2Y^{E+_Llr~9wo-O9^2v!a#cBPS^1={38M>;6*-SeqbL@V{ScCa zs><)03QuZtI=M` zNV^JZ3UL9R!IZAbMJ+C38Xyp~sw*97W-P$x@m>z@E4M5d9t5lf0vjABbH{L_3mGI< zSKhKm{G0#}u4gLf#1uMvk*UR%z{=GJcIow51ubPJVSUo+jgO{dTYMm_UtT-6g;b(m6Uj5aXd-;p0+Alhbs3A9`gWI;5do5aL}EykA!eXelCIJNP~NjAMFP^pKV9ZYdh_DpFR%e<~Fb$oSuUY4a1G zcEqf}G+do6bMCVht|bY&IE@7cd9x8{S@`HppM?k!9fP7?Z!TGGav=>M#tsLQt|Al+ z&SWehchg3c*xvh7StO2|r{CgkF{(Wy7p*zuW<-rgub8`!@po)yLLjnYxW(g^bY5j% zdors-$Gjdj#HGAK)p4V;(%7{LHcuT6pv?`5>_oE)juelRQH~pGQW|imW%@I$)vv8r zuyLq3Q4GXKm_uXd<9d&<)@kBG9T$Pul+A$-=iwzu?OH6#s#s!3&7Rc3a2g#op9Y@H z8T~MyU-sM5A_y#io!0J!93hHhi6WqMkgkX>G>T6<05Mcm6~4u;6FPlsbD+fW|07%A z4qRfdW^|u+H-Ef4DP;Mq02Q&=xij@^sB;Nz(B+Gwo5*Hy)1=gtc` z63iE8DjyW4quA6P9Z1D`uRJkf%ghvNW~Z>D;Ha#yXeJ5%qr#e>mz8yyDt-b27EQ~C z$_zwFz?xDgrCLecSw^uR`X_mYmSo++lFXNe?%}ml#CL$U<>OINBidD!cB}Hr6`y|y zQQ^aRA%x7avU6$7wh(1(ka6JA3^Asb4htUO4V6HYLqXrsnR3IU(Ad3Py|{weld|4U z2&Z-TkC>$s!&CPSjN%qM6A;7oDYLH2r~4`%C|pP#vV!T*qjJitNJByr?S|HJ)_gh_ z@#zKs{6X#6bJ}b;_=*ZSX8-@Enh8#Z4gQ#ReR&Zf7t{7_;-a)LT|T}$8VP(Fz!qZ0 z@v%_gKT2Y;@rS;~cW3|FdY2S4dd6__puR`nq7Vu=;dSERhtli4SW9U^B`$`+c!kPHwCQM}}$H z=&bUNUQ12T5Ehh226uQp|0%76!CK%#izC4Y+4qt{WN|GZa>wi#6-$u1(xn0#O}&}$ zA0#v3cfeYdZ z>UH;bzu$?4JOAgZZ^W{Sw<&B1r z_Q}NOKK(w^GlI3vBH^;2r&&mCrHs#i`HJa@$Q7pej59v_jR(QSU_t2! z;f(YfT5AT9f<(4x#=(=e$k?jtdVY0T)d@-Hxl4r)GcyBm60HD7L;P+@*fL(b5^p0d zZ;7T`Av84SvbwD6{)+0Co$9}Zr3sT}Yg=vks<$ZJ&waazZ`jClrBlVwh|1@BvE?)| zpb9Y_EV}FSSiNGi)8Rd?^*M~6U4r?MbQ86hR6B>Waz8V*`7M`gd}}=Eb>OAIBU2JnKNw|X@L>kxO^6= z&fDbN&&3*avQ=;B7*c}c)jc~)Ihk}LR6EhidM$HU3gv@}1@*uGUaPh{45I!dT&-JG z%T10_7c=-Ubh7bi;m;-1zJ(KSJMdN@#eQfeG3Biww6+o&KT_1cRZ9rRO!jpIo$2>P zw4FCxUUZrP(pED-D-!SU8#-5cDM%_Q$R@EtT+?yXI7U%lC5%C~1T(qF9{RR0r{s(| z3r|+SFwD__*>qOG)A6e%<`QCH+ii~DOppKWF zA}UU7`RTV*YlcYUql;JWZ8y1?+u!UjbGy3vfv1*i{u@kiFLuvoXRZCG0Faxcz_#A6 z=XocQ8l6dQiG%Io{)sZPYi&F65ko^u2$PiHPyif=YsegN&{t z2ZO#X8j@fdu^uJG>F1}}Sprmv?cuRH$b(nA4kP7fu_Mh49fGk?D3a2Nt8}#;tIHAa zp!9_(ZxXO%JX<5Hwck+py>BkaVU9^Hl}DusO+cS4>f=#VQcs%ztf&f2y+7>)nGaF) zAe0x;4U3>56{!>Rmg2#`)#{qCFv;>~>)vTRdm7AN_jP5VcjEr%=;E&BUXXx+j8@?6C&tZag`A2X3wB>0`%ZGID2JCEGboI-;w8|aBZ8&D`H4PQ zvk|GPW0F|P6b{p4%I50+VQA+(DPWsj@3*5F9_k-la}P{8J*!v|`bH;vAym}u^PpH@ zCMjK7fC;J`j@?&L*#jg=Mo>I8Alk0E!~YIQPs;=)lHt@xBSl^0@A%v%f(j{mLK@*O zv^J?SPz}?d9ic{Ri-@FVy|lkz2MA*e6lD?9NJ}g0>6$)hiMig-t9$_cdxD-wzN*lV zem4sFCVtQl5dp^qt1;px9hcZWx2s;1`<`eyt_x#aJ^St&D9*>u!zwXqEiq_jeWT=* zx8b_V8%Vj9qZXF-qN~j7qm>=ZOTzA`_@XN_tHXyuwnbk1-xm@t^oOAtW@ z(D$KN^E)ds7b(i^pnAVLNwbXsE94jz$IPwimb9{pxzi$7_nVd)4nVU1wvQj) z(7AGB!}*-iMw`xC)S4hTi8td^E)`qs3_Y4} zIc@a%5Yc2y4+oakD3jv!pp@Qb;nB2>;PH@?MjMu^pf>HfJm-s)I{&MWIRQ78t0@bx z*!^z^pWt?%u-xW(>jthTiM{NdC%joVp){p*8+%UZe!L-1xOs2SVEXvZCeL{#G&Ca6 z-E)^lY=&(o%$}rsH|ng-Y!UfX8y3L&u6MsB-j|%wX!p4k<9IX(2sxkmBNvmWhpkcK zph=WiO*tJUE_ekcS@+4dao5$G;1V9??=FBsZe zh43e`ktIrYL@??+nu9xqLLpy!+2;lwTAUqfs6<`Q{QY^nacP1oANW=1AT8^(C7PTN zS!rQ7bO(kdxb0=vuund{T1Gpcc>r6qeag~XG?L|LwoKjBegZ(EuY~Pu##TleWL^At z1*xZ?OAaIkt$?*KKqYl?{<#7>X{CuKi~c$7kU9K?cBd}eaqwRswx!Kt`_= z#LaDP0yKo|sHuwT(H&vsS=p@X*0diUYxr+`%ETp!(s#DsD&^eZB`Ok+NCw-d9Bzof zO~lR%)6vq=>EOJZP0ENyNuqT| z&gJe)9QOdmw8xYugy&9J=t{!HLdf8A&5-(pxmi}<&!lo4b{;|SFhV2LV=Xu5`Lze9 zk!h9o2iGN{(JHNPetQG8X-TO^G@}tEva+Mi3|LU^^lL;{$^c;eWcCx7x*JCMKgZRK zRsN@H{O9HI>*hS~iSWJdAG~%tC3_#ex{ej)b3a!+Zu&1RQgw8lZLeq@WV~$no{2oD zJZJ0WdF&j{`uY2I9XDOKeB2-ZvanZuJkNgvzt0Nf*fm| z=6sBobV=sW;2C$45Y;V98=^Zh#}M-}8pc;CR*fNe5?}abIL+m({OD4THaukDa&MkD z)>laQGG>q^nPBsdm6autcBF{hW`&LGNc&LZUZBp0zAn%t@8TP$H8<0WbaNghJq>RU zB91pBr26ms&^{Vxeni>kgz``FC32G~OzFvdCUdZw=%m>KNkU=`+Nv0F0T@lyd6gjT z1~ga{x$#Qk8Juwf9oT|It`?)%F@Jp<-=kCK?#TB@EC|L zgF2gEuGOCYxO+5<82?j#R8SF z&hPDhkA@}w@1hP+0U-cq0TAR+fk(6Q2lHN`?Un2N8-jHQyA>GNJLo>wggTcI#n?g@ZKF{=TFkfG(2)}XA|;qQTx#(B{?%v1iQhUa(#i6Y zxPiQ~R~Y?`X|`CDL!ag0WhQ>~G1$*?<}xmMb(PC52t{eqGNvLBbGGI1YR>dee3%$C zlZyMf@P<;q=eGOHo_ywqFLFS10@5GEcqjl##v-TB%FSOp4`Q{vRb6Gg6Y;mo6D7f~P6A5W_o!a%@Pgaffj#+c=$ z6M_Jh1`NybGNg&m`+#^2)m5BLJVjZJ7&sad;|re6`=NZ>!M*=`lB!!58PIi)%X*j3 zZM#~&AH5f%@?Rt37(R8}cK`bIVt#jWE%(y9{bs$^%kA{({Xx%h5o)}vkbUR4*6SqG zs%e1Larm$0cc#G^mzVw-rd}@|HQ*l&1jxDO8VMBtkE^$iYx?irfJf;-YLp-$43sVb z=^j#&GEzXKLmEbROj=4BL2{I&GzbFH4H6@zVRY9sfA{nK-uLr-{@Xu$ZJ+o1oO7M) z#AP(S9T`0c)>OdJQr3R=a2bR!CzF7T`N?8pV{IAWz5*a3qB?pxKIi|&0MG&B$kd@? zL7BPiZ@duogUC5(gRRj2)49ina*^^<5M}0EZCNd1%qs02Bm>9--~NOBkfnhEdYu=o z*ssAdx>4aRjgSY2JQ(6jLFRtd^&w53A?jf{HXyaR$R^D<%`+6n#26h&96eU@tpf~7 zAhr^#c8R9l2{v!fApL>OvJnd%7u(GF5HE!YV>pj?FbK%uD^?sI{URQr@8J>hcPgql z_kny=+hu(ux6fL-|8h~fRcY9--7Di|+l(Xx-arIOp z?^;|=^(G6+eLGHMuN(b!W~lYewZmmj&zgFW{7p|Zk&l`C>gNgaX2-pbootWW1b6Xb zA01^}6G zKu(9#p9ucRNz${vy!6+F31c|clLZO5MS8wQH34KM8f8>f_#bUpq+<_$&(6D6RG+#H zSiFHz*?@3Zf3dQYQDAB))v0KZgV;Q=P0NJ9ls_mrZJ4(zI2$C5!^7v$4D8u3T zrw=-Su|TMKz-lYc^L2m>#uxB6^S3d2cjXarD=6Msmu^t$FNRw9&1X~Dso$w6Pmd0V zi&mZ1rR%n)|4HaXqhp691q^5!oDO>>=P%kxuY2wsosbrq%dr`c=W{Z1UKQz8c{gsF8ji%+?d@aY8 zkInRA>A86{0CQQP;YZ76y;i(6w0K>gY$mJhiD_&SJvl+3w|ZA;!E5O_VwhzdTS{`( z%RQavhB`HgCX^v;`O~?S6oKl{hqXlt(z+BfdS#ewX3L#+64g0Vrv#~k+S1i{cye`- zQ_q${*pLK2%>9KzSeM?FauOxxqRD~PU1%0gNUU|wj4%$uu7^`Ja5o86!lvu}jcfq@QdQ&{!yjs$HdKPY>(Yq0{G?oMl`^--I&QuW zKz8E(_+Wp*#Q1`w!SR~;+JEO}(FW~9{c)WKba--9^MRVFirkS7k4<}=v8|(k?hoSz zwTrc^SjkA>j=g>;Em<0xwRj-xf|4(zp>6E5Av0>PAi=Q7JxOzbV}Ln9aoSCPg0Y4S zL_Vr15)n5bwPMnG&@j_6?`!Mg^85Ee8K)jT4|i?@8&NC}S81dZ62L=0z%j2`-Anb$ zT;D}I-uq0=d^0uzA_S!GVjZqNTzuOg1&ThfYk0vI^+qFiX>&@?=9PVx_-3+Sen1>= zgjzt6JRE`eGu-{=v2;+NRwNLzRQ1^ZcxUlGTxHPu-<*x(`=qEO|8=_PZs5v6xb&c~ zteWufQ~%EF1qXMnYmwEA$sP%7*xll&zi8W<+Dpk39u7Zd$H`(HN_7P`DMZrI|x;7?dd!Ws-q#G7GsRVDP}6xJ)QR;0fG5tLP+M_I$9m>yRz>hHtOCV z{p+vf`MyZpfPeM7w=K2KS!UnOeP!c#e1slhn1gK#eK^@CYS~f#ku@ICOC9@KIvk)5 z5x28rVJ>U@6GcX~o|EUu`ic^)BI4N27-2Kfc+zYqe%RP;Uqk{X#p6 znrhV>0IJlN3t>B;nZU%*ICVS+Y@$%A#6SjGbxonVWMkLMX`c=5yMcu_s`jxe{{XVl zi+ks_ZJouF!gGhR>91E&v&XE%)N_64ZNZ<~QjyN{+28xm4e#oX7wh~R+mJHR_pD|Q zO9SAn?k7rRhj1dg?N%idGMto z`}*?mYS<%ZXucMKf(-xAmbHbX+s(OaDmcBL!M zBp*b%EWOj%sy|mp(a3Nxmx&#v|KfhNL)`7ogCQ>vp7FYH@$M@7Dsb^i5`D%0fdKq> z;bJacTVMN%pZcATHd>!$jqUrdZ~E4)*VNge0s!(qpfjR~j0FybJL{G^`lV2xK@v1p}W;Eq~>bhm=NB{Z6{fy?E za@;9-a^;898QZg}y}KiG;AlN44~s>Lr$w1;>$(-r(vyi^i#vVyd%x{#vG708`qgT- ziNEkF6}$1_-FN>xmhaROV=!7UxPZ3f5&f`YQz9N_gkwRzgk?@Ey87=jZb0n(PaJe#N zxT)1x&!7xGMh>Flvlz=^Ok*R0K$8COXb~}kjvLAxisVChN2KQmc?l&vg*xBJX(UT| zS-yo}q@@EHWda6pF}GfBvkL3h)-2P{D~3G`1(}{tIyXlEUGweCd4(AeZgyP$Qa?sH zu&5_#>cb)C6OO0w8@*R7kqREzG<3sWhl0kp`+jCsjk8&C|AdRLhr=$~!mC|P+4qO% zPla61SBfMuDI`{L3O^nN&|dYIwTW_-^i_zQBEDx`Px&HS-RJmYvNGdxuo#}1zFpGm z=LB06Xa+bws#r(mTRr~;2X_B7?l6=C)?cPvKhmm(f6<$EQzg+&ef{m5lEPwHX*rtV zbn%siTx~&=Sfk4KwI&gAZUNvY4YluV1en4_07D1S*F-18C65HT^(mzz%tm6PmIt>! z_kx{aSaF|vi`C@X5#+4Pgi_EFou?alRi*QaAM>9lkqllel6keCuw(pkWr!H*$uHo9 ze}$Y#Nxh)apYI`mArkFn@ zjh62>wi3B`5S#@H3cnl;rwB(F84MfP?Y=O0bc=6RLkP$dQ2b@{ia`OyD~RXvoSiYjh3b-h z3s*+H7l$6f6LmzDT8nxtxly(6wYqbb<*ez=^IxNA3yE11E*Y+0BO>zI-Y2BZ|7x+% z+5+U<<*t)(|2-XBSbehVJKT!8+0jpeuKrm2{Yb=mIB;Y>gvwR+DFPnFC$lbBE#aLZ z_aw|j^hR!cs0tsS@TZkN1|Dm0cyOrEAn}5e+qu5rR@s!M($-b^zVhNJdK|j32o1hv z22-EqJP?|QIG~9K^K4x$k9>;uMrG>aKb z%Cr|~$Fg5IUgCG--*jaDKjnZY_oQO28K*z*aQ$y)xM^;ECU6VY3T+#g)BUS1b5_@H zUJG5$xcHq-UK`&^y|?~)Nbj_ZL+jh_iaAW@AO&S4Uf-%ueXX+|8JgI^yZO1;mH3w{ zH6P4fm(!Tl zRT(ksJ;i;G716&pfnnwMbaE#a(1lc!^{oZN#v%iV;gg}x3=TA$Ayg%YnD`64AQ`l5 zA0}mD@@2{fUk$y8$-qEw=6h56neZQIqYODbv6VYN<*Koe~5 z2T$pu=V$DBd{cKm=Nd&R&2DRMY+{Ro-2QBCfvkLgX50QwBp6|!>OnsGa951pht(yV z+u+ydMZhc1M?O+G#{d?lU*$W_Jv5WqHOKU;?(;}S`kSzQ|24I{yVL(kD`JQUX@m|| z9=Bag894Y-oIMSt8YfeMTaM%Q6vb;vG)=#6@n-rkcWVq+082{=k~!eP^2ux`MDx1D;C2S{i||r zO$B#-78xi*6S(}RlAxX;x|w9sb{dwpC7fWS>=b#K8)3D@oJ%XGP(1m9o{9m3d{4TLu+pf+qmhJ;`OD0uv!Nv`mkk*+7f{-2L+RJ^ioXSmu%HLV<{C!r#@jP}no9zXj<~j;Q zvfCJAk!K|ZeQbfS0UD4!^0ueWMmKQ|3fWi?1Y23k$qPQPh#eXxd7dN2xy zMV0Y#!@mF)vE5sWd(sc zq)AY7mP!x+_175Cj1nDordiytQY!+;yVvT3F{?(VPt;2H=FVx`4jPrSt~1=v|68kn zD7fIUiq?6LH?VJ5%WxTLxVzfPzB?(r|Iu@|gl>I%!~EE9?F3vQbP#%}VQ(|ti)6Jd zkGr`pQ=8Y=|G~&7QI95y=qzGdWP8AxJxy9 zfwvHsMBH~E-0DiI{!Y04e%2PMR&lX&^t7eu#J2)rxE0}EtIdi=)HOS``_D{WiPy~9A?lKv zEe;Ay`y2#66}$YC;D3u&#=_zD_|aLn)8l3-?&if1U?5rPcS4vs{{}Wrk6cZW^i+Usgq-e@WU8bUt|wo`@LTWlTGTE6vcFDTHMwM@;$>z zWfAi-X!KE|D&w^<`bF=X&s{J1~BI4_97Y$Rk4#9k~2$L+0dP4+%c6Cr%jpZW<$2Ln4u z_bf7&yXF@UzKl#ahZ!DY-JV#K6ajN5Qm(0na}A=aAx%$gY3%x=w(aWJGQ-8aY4u-3 zxf%>>J03b%H26WfaDl#F( zl>$r~c>yU@)u$P+uSvd@Hb3##sG_z@Z4y(}m%Eb{yT3L%++R4J`~OE&$4RJ;q3`Kr zWY zGVmKX!_{z6`8gG1kt^)uW`tL*2>$JM8ikyPPDF$*=sOz#iV4!*@$tK8y?(jh>R=)S0~BB|8*x#U6-GcVy8J}x@^n4&a(z9Ad;Y_Y!8WjC>}ho=IkO%_ zAOOz)B-DG>3{LM-V?*XQ21yL;p)8a80T(;-VgAJ1|EX55;n8|CdG$&L-E#kH+Wfd& zHbBq`;FqQGDQAWE{{8qw#y=V6>cu;W?E7}2n|8Ejj8hLS?B;ju?eB2On`P77b-+bI z?S^Ut=F+Oa(>q?t*-s^W!v>SGgn z>{X9Mb!XkE08ijD9kv$#0Oq^;AW&A9u8at)+_(vU0lIaqKH+``#g^7414`SNPsIXN z!WI|yuezoUkLKBDZ;}7-a5PWn(Tmu1jD1fB(Q?ISGnQ?)dk*)AiYtU^Z9MmW_CZFi zB~b6-ef47N@g{o5-{UNd_GQ?dujmciF8lQYy0~5;vWIY&ed&%_MXD_Y5*&VEA3Ks8 z*%d^FhozxZ94SZPM9P6+Qsh175I8?@Bn1O7KUKC*DiF%$PWSfp)$Ain!XtS)0$JFW z58ip;yJ0jyFf~4JKzr{+?hy>8X5H=#c_PuC@wu|YlW5uw^j7=_j1k-KdC8LCJk{li`kchluMjbmTFtl`#93` zf1`vH=&N@oB5?i$20#26#gaO{Yay&kJ|kgYpvM-oq^l{Rtq!gJb;gCAWBt9i4*|gL zpU4m+Vx^E{A&1bJno&`7E{otv0hb%!{nHBoNcpg73KQ`h3VrNzZs2#P5i41U9RG)O z=|23?v~XIk(|VNjkciF)<&u4!-gcR8nc>vUYdkhxCQ%}VVMkZ|{pRwvwp+vg;Lx}d zHmrzb5OrQ%Ng!Md)X70ZNp}uWdfA?4E2=jZECCIkteHGwujLSWgl?Vhu%rirk7Ke^5le(+$;wBwoHN=+dhp zQx%zfkv>TDRIDh_j!q|{Q#vSQq$B};@B|>E5Rqn(7DX)lE))nH-J(2;s{DwaX#2HG zH5`mbFGijX1nL50c_RB6UEI5$aDX%>H~jZuzN@xI`S~@}jsGSVghCz%OSRKR*F&&Y z7m>`?HBN_yvsv3?*`Jl>9e1|cSxR`Au-nyjO9u((ley*-Pi9z)-NkNP#eVyv!|mG& zzuQ$K$xn(+%+#WBw4jK$N;=_k8gAzqpVPQ0GOd!up^!4S(n8Gn*K#i;g`6p@XC6Rn z0$l6igh92W=c-fMW%h1}NxR0zlJruz)s`q!pk!I&aimF|T+Sn#Q=6(7x~Jx{LS zU0(?yPR7&VAa$k5!zReXLb-R(#9n3zwjJVremif!_OD4?ECoNkS?V`BtD3WK+FOa_ z_UjQ{aJswP*pCY}{OvmLN3nh zb3@*9-t@V(s0cXBD>^duw*CZ63`?wFYP?AOK9VZ1G@NyQ=^n4wk~yggF=O`dvA0pWEykX*7B8k1ebQU;5kyOI`$jHuk&R`@Z;B zy_WIm7yg8PbKl|}yaWmAuW8 z6?ALdj*^bj^eO3{ND!TB_30_&ahXaZ^fIfi)LjG8377$jT=J={-_l3Y}=hHtA2KtaxRt-PYHj((d56H?JqEVy67 zU|SQ4lVO0L^Vl?|>Cod%V4UkYkSHKE5(_}o%nXm}mX3LA9;G_;0j(WzulnFBX{+CF zTRz)G<$wFyN$uNi&px-^to>!8)Wi;d{sqgS+&;@gZ#Rn9nDPO?#WWo}7F(Pq{L;qy z`?6$~_vtQIP5oTaU0#O@W)FcPxl995U=V7V%&7v16CicvswL1}z#Ly{Pe;cEeq#TX z0#mwOyl;EkY#-GpGSa(oEAb7o)?1KqzQt88 z@+`r68uq9kZeHO=j0ei|2q_}t+^THYDQ{Y@zMQ(cD!x9mn}^-gIr0t^&W7?}%20h4 zuhabzSe|H7NlcuWai0rEvz;?9hqP>@qLPA5B*MfW2heT)9E|%YmVa*Y3*{2wGk)H- z5?awx=W5tw{@)}yKsKDPWgA-2h-Q({VD`wORE z>|7R$6MBs5H_(*`pLU|BowDAissDG{=9WN_bubAWSQKYU>j8)Yk~_h2%)m$PQYzRq zKWy~zh@}AxmCrF2kb8c1mJK}XmSY$Y zF8CeQxPua*wt}wnWe?&aD%syFYxFXLMY}?^avB;wtwK`d1NxoWr4(Ekfv}==JH0-& z_bR9jNHojjo zzFQOZ`9a&{x;Y^?e{gsGH1PvU+%327?<`DZgseT9WRVvUmBJbtp#oi`@V}Ol)68Ya zK~TxO&btmpJqOceb!RxrQz(0q=k8qjndhG*Aid923*JB{a}5_^zTAc&&RjMb!AG(7 zW42oWL}KfXH>d9w;Zq?;^!9QGI2P z2+{3`yyVqrX2rS>?=sxO@#J&DaK-{F4EgQiC$DfEn(+}L{7a59e47wGQp4&+4{>ae zT@BJ;JSL3$v4ab|>q`m=$L`t0qR zB59uOKQxt2ZBXg*gmd58F-r#TWfoAl?#0;2On*3B4gLQ5^(6U-zZ{y3b?BYafI!KeO4g|$r;tuuY)Y`mo(^n3c}zo~mK+pe zHRSw}r^B|4c(h!|h9fxuV9p~1u$8t-QpLjJ8(uemh1BJfeGi+_1Mnu4k%#PUL za0Z3?Un~ijsR>z#lI`n-mi?#3ZiMGFL^1)cu(ivwE`6;r=+Dix_wcqcw;3=NLoGTuBi%o^rz9*|t^9FS(Y+W^l$EV*(_uHYn#AQF>2oqd zn|bC>*zHqplY)MK5KeG9A=vAP(%mH5GqMQHo3RP*Ou;QNii9^vO5#Am0GW8*`6&s) z@09UzPo-rfqSle->L43s(}dIH1da!fbt(r2@d@D@0)H;`c-&Ib zG3^Nj6jahD)hEK69VGIDF%abD$5pq`ThrU59>-T{MNYa(#rOpY2W+ zL8NOpm8IQ6^Cg@yMz_vxCIpxC!D@9%TojHv>La%}^7z3j4K(#|sE)Vk<%eJB#U@Q& zJ#X(z*Y(&f%glI%CsXf+pW7w`A2alLX01930;HiifO-6Yuy@Iy{jmsxF)rVxwKjA- z2$yMxg&H~{Cr;Gdbiqnt*-^0k{(EW zF78rXP2lA{7HC_Tn4?`p(|5c6e~Ne$sQF^sIFJcoZ{-Rf8K;gas{R;CCv4+5tchky7(wAxoq4Viu{0b>j!$n?;o937HV_MAJ^~v@!2BrGUM0-9XYnbL)If z@+9JTCf2k_b9=o3gmqehUKC-hMYL=hd9u|&m6Dnc%)x<5>J7svKRac-j|fqmTKA{c zO5yle@u7!7lTW7Pjz@o;?WYryl0$KGYcG4iM-i{w0pXc9IsK^mRBOV&|5C%ZNV_Ea zcGr?fbobjvw*TGD@v#3=;KK)QX;@!aZ~!faGwYxGEF3m}Lf_b?oT6S5G!`6P?DrA1 zlsL3KmZ! z22XtiQ#8&877GiM5sK4%(x$S3)e5M{pfbqWA*Ni-uQJY0u2@rCY3-qGz1ok`-rg!p zWXoSrlFiYbh?66F|DF&>iwul_rvjuWuDzyks`e&y%Zyqa6jCCD#D3K=YbghP`3T_b zoTD(j>Y6F7`g>i$h{v(wXTNVQk98`HTF@|#%eeIhU4c-e6)Psf_v8++=@Joa8GVIl zA)>9^iAt{eT(PTrr}_4t!tmf5&+fLh&hXP71Qo&s>s>3%_&2B;J$b-oo{FP#v?WBS z9Hb;+`Wtza_z)Y#tBH*qzM?~z>pK6mCN-FiXGRR)6`T_}jN)u`{dw(TWyu3@-z)-T5s|z+*Vn}S zV_88%$QSD2SJjt`SWmEi7emiB!a~Tz&^r*{9cDuztKL7RRQ%WiOv2w=#bH4=4va)4 z=TGvNyc8r0=NIr^FDvWpPs(x@Q0||UL7&{nIFgU-qa5jeIt<^8f+6<_M?aumPXez8 zY0GJFxy4Z*@lz-9GBtJmIGwhupxeK3>4s7DR=)M(uO~&j(thr?W~88NQ={%8md${DhPfn7CU=K(I$?f*)nI zz+w?0=&cLmHDd;Y2v#A$mvIo80iVJ9nPwyrxIINNnvc-h z`{nJFGra^R}s1H>pvIUr1kmn*N1{8!_lCn%Sx35u=ii$-BSjhUHk~TNVFDD(TR=cSz}r9n z8hT{N3w|%P-~rviLH7k~^r7gG8XQ)YOL_xiO0{oMd#q4)a@c2LVv<*ObF*7%B}vQofM88lV(j9n+*CXt_%4 zb$bq7w+C-jn^%Ui|Bdg~X-1AeaKh=lHmipvycaw-#1GoBPSe(F6pl|iCIDrL9Gr9R zl0$#=I2qloH(za?jfp#~tYz|FF;OIp50&1-HbaOZOcuOw;A?2r0Q+P_=kc%S+&^B_ zZ?tHD^l+q{=xubYo;+gqj&g7}Zo!IK+P&dJF&w*Ht)Ks_`?{fqu9E%N=Wo#|M zD-V?$dGY=tUb#)k8+Cmt!uWrIfaouI@;4Ba=?gfm{QS$Rwe0&k&`Qo6_Wu6Z)N|DHE6ZP-r5C7{OizMo=1=}tM(PdPJNd&Yo!?`a_z;;`75QFEfK_m;N z$Mz4x@r07r}}TnJQF zNZeC>O$8uEt*PDv>0k1b093gm$zTvm5Jeb&x<~#*x^@Yj@8*aJ+Jncqz@cwgDShmyUnW8Y%SiUm4R^FGy-gbYx|Gl<$^0qol zGewAt3WvVT&XRash)LLIf$QsYex&x*>FO4r!+}?XX0z*de%tK6P69Rav4A2!u}%MS z3^C^X(tG)ReQ8Tr6v82$jv$s*e%goozuVz0>d_wRmd>u%mAoI#u=hNFwc|rU6i>&kJjc#g( z41Vjep@ULtjSP+X7JA$OC06@I%NFa0gvfokh8i@xsr9%(&_kRr-LgERAs(s?ncI}A z2kujQbFiUMD25O#jQ{{>G$mEaGz##Ju-9^`2%aJfSt?h|?fN3a| zG@3#=7wZ@2O}>_udbYE-wTGOAZtq%Ajs2NaYG`U2LmI+kQsj8y@CT8Vlq9@5Ec3^P z(rL`%zSqHd4&*q*OTimio}O| zQ1VSOMS41d8JMgo2E~X}jxJu=tlIs=#ycE{mthDQVN#Tp{VG#h`9<-YQkZjo$=d}@ z)GLQ)D&wV^sJ&g*(-W}%*a*-2i;+R@m z>-JqAfdzjz@$+!QXQ$*Jyhz=4joe=!&b}Yu8y-Mg(^_AWga7p$3{fbl6+XbGigj?4 z-8dya*F+o3<8&35!-&UizM})Bn?dzc8q>7dbyqRU)AT-O8ujNvU%g}89v%<%-dOdI z2)I-nQ8@-ZSQ`4)W^aiwCB31fv8lm;7dxr@J+ER=qjJtj99hYl()4=Sj*(H*@>*?@ zW^q$>iS2lD`4GxkpB1W9)@dMCVsH4JYY!z|3+8=7gNKz%9niedx5UO)tLa?d(6qMd zJa4!Zp5@x{+5gt1K=?l(gF$6{*cavEvOeIso@yy^*{fErUu$ED4#ZI^uT;K!+n-vMO#?4zzGvD0(xUyZ%XaGOs+MeL zWB)%uA#(U9L2W2o^8PQPKqu76oCfYcLGrs9JL-uY)D71d`%lH4hS$Blbjo&H^P{$R zT0|pzKi-w!5xLF}%ugCZj*gY})7^J-b92!`eN3SCc__A&`rv!*IX+TDfqENh*|hww zf>*~y>zpc_lc>Ien6h*cQ7>=rIPkVpNHo>dY{i-dF|h!O!kd{DROeb zNUeWTNqf62Eq~2fzpU~4AbA}W9i3mh^C;h$B*=0h0H7SKM8QbnLdnF-2XgZG;7UY~ zaO8d|Fx0j*E}!+%VNRIi@B0TonV2dwIR6yu^Jhcy{&FNoR;N^#*I{AkZtiY3Y%fEq z6g<8t?DMncAp9sz#fjZ$n2jReV|n7yaJH-8-DoV_s0;8rhg9V_Dj6#0a$#vh#IT?egI~mGBLrWf(5L;1kT9) zn`F}QM$hu?$#HthI`MyiCEB?^Q%nueDP$lQ zV39nXZ21v#4gk?>Bf4hk=^jrGmC=IZu|L>bSy@>hsbm44=%>lYKAAKFBGai2N$KC@ z*%RatS8rkp$;20((0(K76@2)yujnJG#N8=-!LUUV`QM+cky(F#cm0?{_uyn)Z8$@? zx~0VzUAy`_bg)-P3DB8xJcU%;S)?A%qFwer?Ks=*XZiGzIcXt*u+QB^b;Zoi zt+KnXd#|A(<4c`G{qYP^`Rt2d(y=^)46}q-@4{k4#VQ^=h5^(G(A{MyLQp^K5Qxu} z8>9bomFEq)OJKXkh`QsUjW%lAi6Cj=wY~v|3iMI*^Um7eBU*OUygtW~wzY+5VZwMO z)Pu;6?MoD~~f@H8q7!FPwe^;oN+0h=oQrQYwVq`5dNoHscMHpkr z&WwEqvpm2W8=emB^{rmpD*KfP->H4%^x$5@cp(y)cLF;~_3QhB_ zB&wcGHnTgZbI;b{kO1Xy!fnRPAk(GSE^N2z!@ftzuq+ZdGh{yic>x3V*E;@}0ay6TmQC|(>)bDkjS1#oW6!PKWQlx(MR5*oWu#f}VXQH9Ry*Rsh&UsKHr9RV$ zYoLd+L!;xfP|aoAD)1NjBP3K(P&od%(@0~tPxnFIfX8MK zuFWdhR4@No9@ZUQKKwi~`-vtY>0!kWZr<|z4J5nziII@sy(2}=IO-0d%ThFB-p8?D z?PcoUHCGnJ2-JlY?GMCuX7@_T;A7PhDZdI^+-E zzwE^_o8ee7YyZh5elD5(D5YomGtpc~EFk({`5>X{o;LUXrMSdQ`R)V6;FVgvR)K7}NDh9rAj*)h?*o=^Lg3T=eGl9t!gC z+fk@NgqUP>tYl^qB=5X@uDv?V+U;g;tcS{-;b*qp!vyIaW|J?Hoo`PUH%@;;3!jp; zDxW)oY6U(Ye}$zAVkRNS9U9+V*yKN2{`YJ=p}K0U-u6aD;{9~m+u(%9`$_u%0C9nh zr!x}tYL@BIQP1sar@oxS9(=5GeBO3DWS?C=)Gng_8q$^-dE^7t`dgp2LwC7YYmP>v zXA-N;!0uZJu~^bcj*fAnl!o`&pQV$_Eil@ycev>r6o{3Il1mp_U*T#hX8X({kPSx zf}Q+WT&%$HJa9E^cS6h`ieTHtM{|#wK=PGd&v>n;U1l8XTm>8$hDqRRLufqqyQk`959I z1J)bR;fI_yz{69~6jS`}b~;RdJ4pd=@F;V52Z>&=DaJsGnZX+JX{~xYTQS#EnoS7( zHncT+>Vq9*)fWimkoCffLH^Hg;(N`6Gs^+^T^HPi9ziuxmp7u@mEi+2 z3I}>b6e92;P0u7V{B(ZB-ieofkQH4*rawraz!YNxcl^QX8vA{}qWdMP?X@u)t=c!bW4Ewi#eF$aKP5dW2h5bh0ewk)5>t-v7*Zz~VH>fw_ zaYx38bzkSoTWl!zOEWNlHFfobg4FlOQ{PiqEL#G_`M?SpLAUMOEqA@UCTqs&Q_!)UtX*L`J%S|wA{Qf}bN5A8 z#vH3XXW2t_varC|z*LeT?Yn-D%e^|4Mx}_D98eqWuGWA)-r90#Zr0J;^d}ZbD<`vw>mHZ#pje=Y3u6lmNiWz zzvG1qg{BH)(Sa<{==2XrM!^ekZfKvX&vo&XI58QCL{^*JCTn%q%S`{#7%iu;m4bh~ z?E5oGR=TOk;KjiNR2WD^6ke!FyK<`TFF-#KBhbLGDRXIeX^4_cAMMB{C^0>kwB16C z0;w19;+Oi|A?8uCh`DA(SjgUAC&A0 z;dlelO`+s_k)NWL2rCqts(pR_5g{dKEx*0emTlN_U{Kiqlc;VEjzh8bUZ_r~n_>mL zbDcd7aHd2A4iXJ&O`=+0)B`Un^o^q#&$Ue?5-AIqMJ> zb_NRjDTb0=GLXUkfNfNQ-xAAJPk3kanw9ck1+2b5q0;U=Lb?qn^W zSY550q3CcJ1Dwd8#DsnPj_xqhuZ;5i;$ntx%Rx(+U;FaHMBQ|xUJrP_Cw!O5_FG#V zLhG*r#uQ_Kvc;r+bl@gQe0rt*R}4h%*aiRJ;o;y#^g#pn4C|JQ3}wymDATpoPlJe? znr^}>ZdB=ZOF#lFd9+^kxq<3@MQmPZE5ZprvwipA#UuHQ=WYhERaH_X2tFEGy?^%n z)yK9;xENLc0SUbt-sOMNS)T%aTuP>9AXmVMjwS9*=CVi<0Dg#Pd$h`GO&W9eGe|XP zz_Z{gBEGGsol~pCAJ8bybmy-|WefM4_0oJzUrVm-;3VHxWQ7n!fUoCAX&ug3IvBMX zaJGZ_#N#%=Pez=1T;^6aNgp{0&>JCcc>`Wjm72$#W16>A&c;5hx<4p;e-$_~#9*KU zV3gl}{8$nmZ0NdH1Vi}mO}^9^-0>D$kWzdHSkGM=WYcYBQDVIa(dtaxSxvplq?g6S zzB{K|okO&LN!2h+O>Z@wqp0$yqp4J?u*8(S*Z$)=J?Rfg5i+=AY7wsw8*P`yYuKTu zq`})Uh-&!vEC01`Yvq4R)++vof51`Q9N^Gd>DAo+dRU_1e6een5x<-v3zlXrv5ebY zf+wPFM9>hl#gSw2E1UrUu|5=^0_xurFjER*WlEN}V*}%mW9t8y4=k4Uv15}R+ z`bh&8dMHz@9}fOU47yoK1~GH5sD~%@FI|%r?34rgA`;A&M+eAp0!}0959*kr*8M~M z55-hL1;g4z*kO?>q6;s$(SQnDii}onW-G2j@6Zh3#l23%>5e-7LhjiYg`kN8kWa?7 zlEHUT-;3&|CURJ}4kh2o&Dau-wp6a~mVM^S?9kg;DfLScBw_V|UBmLbeOuX=VjzSE z(Pxmj3(>1?E^BY#OK#836Rw~Qlu?R{XEYS~?7=5bThF34gMusT@X^W*t3LoAccuRIeN@D1e+Y2nXlf1U)lw4x zxM$+U%vIkC+-~$)|LbZ$5Ew_vef#KY#tPgo{m16yaG^1HdblI?-n?Ha47TIV(6j_4 z7q9KnDT_}dHeQf`?Pn7;ERczQk}m{(BTF&NChIy|U7Xopow2>_K2-r{xoR zr{a{^*q27_BL1caZaJ`kpD}T#aI8&*yX0u`9GPmE(Co% z{K#?cxe1vTOy&Lbs|y|Nk{zBou}~|eN4#;DH&T^g%DuDG`~AxEk&5h^YlMKlMb)qg zL(7Db{NLoJ`q;O0MTCAf?Ud$@l?ZPXmx8p@GYhyr|oOgvTSixS@?W z{uYA}#-nXvoas<4X*aT@+}XNyU6!2-72R$&m$;+BWy%+wa+8E&>OjH^8#}6q8xDjenxk*91Trc2i&rI zYXPUT(Yw&tQTpIA;dIx7leQR)*l+7N6@{GdVCfz5X=*JWu83v5AJx%1waIB47(Hfv zbdg-s(1wqdie@Rp6-Si?1w&RIMmOz&Y`ewR?Bu^SoEHhOV$Sf%rGh7HC;ye$w(MKV zf5_45e<@8%v4gRgr*Zmy{}R`B@PkV=Pm2&_se{sn%1OJ2B<~WMQCI4Z;b1cy#m5Cl z5rL4!bcww0QZgznt($?g%8cCAZN}f>`%L|-MpZwb8#hnQF3#a*;`FM6H=s1ZKp`PT zYXco$cvCt1631b&7eyiNt*=ZN{!J@KOTWkBJ*>%<=u^IPbTZxsfEhNalI8kb9T--Ry_?FbY>j#>9{umG#0RKwOy6&di3gf{1O9a zQ1{KBV#2Jx*xxQ*C?!YBgfm-gR`V%c_9R8G1Gl=eTx_nISgLTJzsTO-gqtBBntVoh zKkz0rP0*m4tXjpda}Huhov#|BgSY+W$UcVr#ZlL4q2F*kVBNWJmmnC3IT!2X2hTEr z{gUtaLQFH8z#p@88MHG|4Q_VvK3G15;$G>X=P(y*htm0cdd^cpw51*@GxhZ;%zXCX z!_@-NDrZF@ZVUNEc!}>|{x2l7_?-xSAb==|< KrtrvLcm4-;+%~oV literal 0 HcmV?d00001 diff --git a/doc/images/fcn.png b/doc/images/fcn.png new file mode 100644 index 0000000000000000000000000000000000000000..69ec49338c04805c6b15d02bbafb1fa5c23ab3b2 GIT binary patch literal 51573 zcmeFZc{tW@_cp3DNQESoLdsM~reu~VnKF|x${flp87dMgN|Jeqh>&?GR8pqQLS@K2 zlX>=@ck1~)$Mbuh{l0tecke&;vG3#C@ipC_`}4W3b**)-bDeAXC@P%WNlHUXL`1an zytJe;5z%ISBBG7TBpdM)ZKuO~@jqf)iSw!?BqZN|Ug*TP`|YGO?NqFd?HmnkjEGFE ztSyZ=Y%kjw8CltyTHB2i7mE@R9U?j}d0N#ee5B1q=lCcVs{HT$ZjU%UfbPvdbm3Cl+%87FfNs>keRig-aRl@PKt_jzUI zN3n~^Cti7;HbVNOJ3+FSBT4iT`~0jjV3NpD!)F|KGgq zpiSGK&-M5!$xa>o^eL&=hoK>Ja&mGe844dsiI~>{Oo<2o-e~Cf;>>XI+Nz6@QG7^9 zNO7_Icza>%I5Q(-xPWb6Z)LFV8wc05l{uH`f%j@c+FmYmW3InS_m-EJfBpJ(VPQe$ z$+iCKh^Ax>{Q|r4_I6QzevPJ`s$0~l*Kfj<+c#D{RcB#(ut6t| zMK=Ye#$u)XRu}rC@NVn&xA_GH1sZ939jEWBg zxpozU(}f}Mq>|7VH!R`Uv$KqhZ*j4hZPd-+ z0oSEj1_p+oKY!vr_w3oDr>95nHXrlwA(^hKfx%C5vE@q!1~F-AYeP+JzOg@k{P^_X zz`5eEpdkC@`Rls6Z9jhKsHijqv1w>syvSvF>RP9VA}cRHe@FV|b62lk)zZR`LjnUE zZg1YcW5*Jf#^uxl!wr<{>ZTbBvMeL>ID@m!TWIK0qu(G~RPj{XjX|0c! zU78*&sSXGYz0vh?8@3fs{HyyjztO%Xe}ap_Yjgs(gq6M!DZBvlw64gs>6MWv^A}r{ zR#G)K?i*}Kcw;|&mc-}b!xud47c)MVm6Qbr*`qU&<~`gIA3+kt_BnVC~>ou_PUZIAj=l6sZg-8nTmne^&a94l_uYNGQ! ztlo|xJ3IS~H`#-G_oDUt`uoXg1+!kgy6=|nFsjAE!V;q>UChqOc|lrw0Ja?^>N-2# zaUnM+C+F*zFC2#sbY>1k0~`Y znyrghp1L`Yw~NTiR=#l@=iV2FTFZ%MyA7)5^YRCGeZTu=FKhwpFSzMbDq`!3aU zea}N{%F74ej2HR&_+VSfNJ%HBrjlR2d{mudS{IwD`zBJr_UwfV{^v(<#8p*QO-%=| zcC(e7#d!qO!=Sw2*n)T1!-^V~f^6ca3J6m+$Iw`SJ`;~TO zTQrj$JXKp)_h{wljfE{2E_M6~6C7Ow19_L;-wp@}cyNHp-j8Y(L(F;Mz2%JAUD&mS!Pj&wzXPNF3}A z2?{PEnqZTr`)=0XzH`TY{-<|OPtUh+T5!+VIjvPy_bHThb>&ImFCRojb->9D_C<)e zoIHNqs_Ge+XTY~_-#R-xKYsk!_Qnx6qkr^njfU|BR$bqpv6wTXRweQ9WD2A<%mcm=Xt z;EuUv`8_ASa+i-}vPN3b_3pE$Prrv-2(ls<@w1{@5iZ3AI^ZPW^HNh&;U<6m`n7rU zW*p77yz56IaDopVJ9bP|w7t1m!!So*Q8C)?$N|fsY3<|3k9!7u_#hENA4gAlQBN;* zlKJI7q6>{IBV|xf&@Kr(`?a-*?rBCig9i%BOXJ0tFJE@Y&h=e)HB3JxBowAg*iP!> zIs-#PH%400@7=qH<0fU!&&&H{USbu#kNZHm+fiyitkCzEGv2$7oTZ|oIw)v&&$;r$ z2g8-cnfUm4&j5te=jZ$%((w64x->+Jxht$u(9k4FZ|@CgI5t>CM@NSzH1@t^>M}dx zJTnv&5fPD_%U`(YF$%BxUB|u zR$de&n;t!SG&o2&>X$?(M_My)-`=?Ba34PO$Sz ztP5wp2J5@!<1ph_uU@?}Yj}ZUKejN{f9uvQdwYAl7l>e}@g;&yLs*i-_3QcWYpY({ z_hO-~e&k*M)g^`1Ldb7vX<1&r2~Q+Z=Chk+XlSUwVKnsd<4G7RmI&dVUclz#_wQj8 zk6~D0VcMFS&mtmp)zuvkOP@S>VqLWG>D@c=Llke9#@-<8lhX;;HaGkG`6Z{M5S&Ji z<#$9m@0|zK)ztVL$9`Z3l&KI>@-s5HU0jy-^B9MVxTqhZ01jhdVKM1^cWZTZ6*q5~ zK0n!8nQPS+dg{92^As6*d0b6FSy@?L{tKQ0VRdP4yaSe*e)+Ri%j=&wPMmsgPvH{) z7~s>+bwnFnzRcscJfBvyIMkYH0`H6?h6{4@^ItfBUchxROOt&Jc)>56%$l+D{F6Vi zzmbK9XR1bQ4Xf-7D+Ja%vZA_i=IHU`)nC43=jJ+IyT*z$h-jBVc*W9(p0fOwg_|+C zas?)2Vr10X(qcFEBV^&Knp)7qhYue;QZh3$)6)Q)Mn>f5 zRpsR$NHcPuJZYs6)|)F+gs6nWjre}xz=1(Wgmo=d)t-@&t7c{)SayAVYh26Btekzg zlk_j7e7E12uG;?0s#jEsGTVsIxTN&}2L}gu%Kic+VN12`;{TPh82pVY zME^_4$Gn(fI?U~qh{y`IM<;i)7TG7-CwF+ZKDitC`(LyYHk9+&v95wWIx-3hscC5% zW0?g>mcF5?8X8--Y#{^zKWdJy0uFINzXaZoGm&!~>H7rN-*{3T^$laZZm@8(uy zw?$rL;w;m@%P#EwRL@uESgHs5 z-o3sdI9FZ%yf;Ec zQSsXR#82!YG4Y)c9+QH+ys{c$_toWi*`#^Tu1^hobd;B@YA7_GL*5Abdx0UHySIQTp}HK$JA30~f;=;%XY zV(z^aL4tyU-*c_;ut_XjW8Z8$D_~=Yk*97b9(@u-bvmlIM9YA9gZZHCe!6Y1UF-U*FYr1*X8`qo$^o zX;QQL)0;l@VqQ{(S<;6OAFxZR>gvi=L7|~T{rz)awXOH#NF6xf;^J~)B?_+Caj?F$ zr6un|lqe-rPj7GG+Uin-kYjs+{V)OYx4&Ijnd`tWMTCT0rux3XmC4Du@)57e0E_4AA<;ix&v@_O`a#xmLQ*Emyn!#lEKL(;qx|kdo47ZPj&f2kZo{ zFA^Jacjo~*(VK~vG!6>bTv?dxB{ep-psl+FSi)!Vb$POqQ-@#DrKzdO3n}5}&k6DI zX0wa}Cr$*c9MEtf(|NJ7G`F@qRWm{*CMG6P1`EqGe;g`(|IO^dJR5~5(b2B1536r% zyFU7O27LS|Nv2)8&4fT4^;q73t^u`=6(!iCzA*LomLmsK~mb2p9wDDe^~3uQjX= z0DyT`_5-(w+A7Xu<_nQGhre=c=HbygXGb)DL_|b4lcn5x(N?~jT<3+`%A$NI@5|S( z1J1{A&Rza=|L1G9p+oDI+nR67!f*x1N3H^i7zxAD>}dCEweWx%DpG#0f>i#dAuH zuP$b0O}PI6^UzYap||J}t(eImT8mztUccS2~% z$#I)&bCf##$gUa$$a%@F#x;?JNPHy95~AFWJrZDHIrk_`k!D3OCPZIdH{YhTtxfk) z_|=vm3ZnMJw{MqmjAUhG43!J)1|>+4ivCPh;y7(Vu5&um_{%q719<#rR5uoqlatvT zA9QWEdwAPzp*L9R`PqWt&iIAQ#!-ktIUz#!XV&)5M@K#_3L}SXG)#c^R@SV4aw?3o7 zF_kC-`WOrb1WHd&AEBkjWn@WO-#Dfn?j_&PEy+j;X5;1g6!Xn*C_$Wn_kkz9C5}|& zEQ#~-{4ZR_Qz9lme*q6}c=>WeeSJkmMR<5PHFC#Gm#*RD;Vui^R=)lCA7+K`zj>`*Vfh5 z63W;f^^K3J|7K#Wf)bGFaX))JN=3UJpcq$6~bc) z3Z!$N5rENyhYzVqy{Oa92CUF#I&%uc;m(m zn@0LI`hFJwmoJZ63wcj;CP`>8QjuD>70i6$?^vGi_SdoZ;%$l)u2?DfX;O3IRPki( zm&C+GC-68pS3-_sRk|k2+xs=>&Q>~-<(ia&KS8)kPPWj~YsbC7I{EDe6hXx-Sg!U2 zUTER8wXx~y?gn6LYC1<_+5PFk;Yas|)dONbQ&~|l z5Deg8Z)1a?FLxK_i0du+aTY}rXrJolX2*9P8>pmc1#B4E*#TrS(#*gT{YNlHuWNYTz^(@3rL@$&L&cp*PrA72U34|1WDs(q5bW$)g-ER2k& z!0!kN)r9eLB7Ki#jZr*xwdDjSC+CfhT}kO~io)dX>eOZX?UGu%+v84~Xy@B7G4y<< zcUNsTP@#&Sh*lY*cRX8pFz@!f?dfRNA?!7lAyU|RvND*n$*qUf^k?9?)sT|h>Yp`g zy+GOt?z_gn=lW0dtvc_C6qeC ztkmR7XwcKA%(|6HYRM=pzkI*Py1NrUoXnK}v-EAMwZD!~Fe47fo9;j830E zy=&L5>B2eW9*a0D+ZceT3FKLpSp$%8?~{hwS~x9&=Kv`K9#fc``|-u8wSl_lNJEx0 zYect`lE$(Q;dTz3ysC*i1?CwBpE6ij;!Q3jBqWG_AHzQ~&nbfSAG4XK%Q$^J9uhm`p7vGOQsWmiD5q zJLIC5(m2%ufq(IEJjv19w{PFZAaY|mn$;X94`ka(p-fG7B(fJ_ae;?3)9&BD-x9Sr z-Ee_}oxP>M1FI5Z!F4COcTVMnTzXE9vD=&?)y(pOEr2+Tuz_>W;8v;02^PUaCbdWR zJa}{aPBrUa&hpCIuIHi;$OPk8uP0szzq~Mo zP=oYe7ogJFIX^7y0MCr&w00HfyVaGY27@E%ns1Sk@bmLC3m#f|z|O=^G~H7k2*8Ed zH#<;w1Rjxzsdv11%}qt%i$8Vw-hKO6Z4bsjCk8b@BXT24^5mj}60=S_tAky1OpF1N zZ%>LoIrYPW(VflPMmUqf25cdxs~QPY)YcY4!a-o*R*{TKkhmO4Ih!3i;4y7~tu9u| zlVbMxBfFlvE3Ef$)+r@gYTTS~_{7DDT*vBf%#M_kQ7tSj<(f6zb#`7^w3jGDvUSuq zcDzdyS^t~;Af8WxcglD5N@{Fw&b4T+2Hy~o2WS9pBR2LhY5|>{mwf!IqQpAhz4I6h z=bzbAc!qfteEcC#$+Ks-xqbAw-c|9Rw1RA zecqkg8O`(JLR5RMwSM2%1NMPp15XTBYO8+lc;)Bh?HfnI72<3H-a6qfZOP=lv_^W& zrc-yX+7b2}5OhS#>C{%M)$->*jJ*3rN=n$srIe<}PAohBV0TXyR`s(Z>oc zqb(gZ4FfLYIJ>e^L$Ry0v~azs&zwDb7U%f+5l(quGS*f5+aH6~jvhUVdYh5SJs%%~ zwF@z3WUQ#Hgjc9gAb3%P zs8s#p_S#wm#H~L5E#JR4fBi}*(8>8$6`7lwUUH-3H4O+0`wj~q49>Ogn4OqV>@`T- zP{`gt)RgQyeW%N{bgRvl2;N@J9)Fj>Ev|r@s4tyAe;yC2GW%N9W4~RjrWmjL4`-Xs z9dvpW2We?hF1UvB1KjH|M@5e3y%*>{Bvk(_S0j*x$?7S}r5mynnInWNA_RoT_tWV) zlQ2xzFCydPUGdN&W6_|iKO9lAaM8ff@Gxk;jT_$VG&VK{9!5^*836ul!X0Fnk%k?x z4$dBKoUNI7=nw@7XgRDx2$hh~8#th-Q`Zk4IdVq zH3Bd=AVT3FP+D3lwmSd(hQXT1~3fT+9oC@TQ_Zri;wpyHI%sL?_U$ZaHfTxi-)J7 zuFkApGJDU646Aol+r_xuM|}<5Qd3fPl94IM$^8H*#$s&Vcv@AJO2`)31h@;No=e!b zuT4#(t(pA@hRu^w*wx!EGGGiZp+<0Nb{iIec%b{8KQnG&Pl`8QxmDL8;ll{c_0yNl z88~bwCvt4(uc_tVm@&V8oy%2)ZWPhUGaxQ5?*9E9}Q?)t7E#uC#+u+-J?mvJ>;t<8DQ>S2J`}Xa7J%uP4930$? zh3P0>1K&N8`$8_{L-oy(7Ou%+Y0@YzF6O>qKZIt<=JevyE0l=A1J+!Ks(!w{ExWCy z-X0}tY@Fq%;h3D6iC1Z-;&{Bz&@4^A*w|aOXCKxV-h+vfkgFjAiMTGRu9}35?uh5O zn9&2&Dg+uDOt8N{`BDl_Kv$QcLj_CJ&BLye)6LsZ%>ZV4vU=y%LWVEd6$%N2Qs5mb zO3KNG3t~t&om^Zv!4kE+HW<0EZPUixy90rx%f*i8e03f7pxn4|BfKT>KR^o8s!HA7 zTC5Qud*!E3#xp}r7GGb1cM#27iud!|&fr<}`~_3p8jI0a*1#~X9vO{xmZhusrg-C(pc&fBmVoqA4e0B8hAUeOpcOp?}#0^dMobR(HTRiP*01DW2HEszPu-c?E^+3}8K@OpW`^v0EA}w+th`A|OuV<^Mc$G1uDqnb`ezAv zi)t&RdVAQXqYS}81NkK)wdvx@4u?;Xv17*0PO*_QT!FJekqd==BjthCqu%~%%954K z-@4*pDe8G|-ZVEhzP8@Y?Ah4V1nzZkc=&T^>7LcF_;@Cve!y3GdHDmbpEmTja50BY z2UAz&K1*-G387*p0H6$wftx z?EwnRNRfB#GC*>BbHS$Xs81FW+q4D#g3BNeo-Lg=t!myz>BXb3&K0xa8)~if_4Qz~ z#7t|WQP_e>0Ejm;(p&Qc!p4)a0B%H1VXW5>j%}=0<24I!r zKk_eISrs|`G92`G0d+9e7Pi7q|DdQ?+*O|goEI6r=rZbgVCSbJ7_;C3+?IcRZY!`~ z0w=^Iin3(=8%OKK9V+1=A+1~KZxXmwcsrg8)t%%%pW^|vLHs!k)Ea<vA!Js#jp3cTGfzcd`r^*+BtHtZMgRG1<eTJYR{Hrjw8cWm457&*9!_O=E|;XsN%{qwB#u14w%j z(*VIxX{=bUH1ZmJ@I>m#Rax`fW0ZbU&A;Bi{I0iq_BJ@A3E0)SbLTRxQ-$n@o`@gn zY|6O(RKS_B`e39Z8GY(8%7O2+VRj9dx108rH&p5K^U}@k=y&%8dD9Qc#z-2q`|_`I zd-n!2$v#aSV-52QWF!BXYI{nI4a6BJKzZvk3PRwLAbP+FIbeBHO3F)8=MGEH=V$Gb z)w$2o^a?YeBbl9jp+Wz;T;H&qj3~VUmRk9vEIKW%=Izqrj73N;OV&FLI}~tCavrEu z1Dn8G5I~~OYJw6Kgf_oWQRU(^YB!>wpg7?$(&9}{e^M*rwm!>=-!7o9GU@j1+fbo! zMWG(5S?pFQ$wJ~KQLm|U{!`5453uqgs3zi^hlPgXC<~uFd6J3A2w($W03-qYwdYuB zJ2=e3I|$khT!MH8be?$=+(#0a2M2KQNa06wJB%HICn_iKg3Z~wQ+h5irlaRG0@)L4&_pnw$0)Y2!NGw z_cb;prM1-+$erR2oB+H5hMOB|s)SIfz1?lHrySsoZpoR#Tx$FS?xY=+UPfkS0;dVQ zPbqY*f{<>$uHi+Q70cf+c#tkH{B*E8yFBe2|IPvwIjV*~fgx(T(a9^dB|^C-&(@{W zw{!NkqMBMZ%DN~9=oGt&aBx7nRQBV?GAOAbQS;L5cjZ$r*tXONc~2#iUIyZX$s%Xz z>+1u;0kVWH>8T)D{!3N+N!DM(cRziPjL9pSRHVuV*4@LV{sUw;b_>|{;Hhherg%9; zJ0~j6TyGW#kJ^LNjccI5Q|!LBiX}xQF`BOsKgO;nVX?>~3|DO+`1Pk%owGjw$U=nc08)Me1_IzJHu$_cT|C2ASx6(!OW zoB$*Mn=L{U{r&x!QD@TumT_dKF9UQ>LIlKp6x=2=Gc%GGxe9>9 z2gv;&#LPWo<>U-*)wrspq=YpNR!HC<6MX%OWwq?_kjoSrycStFrbcDULcW`(y z-jVq7C4?Zz7_cyt)6+V+Rx*_$!wm=TU{rxGbPvv}!l!k)XYxgpU*7NoBJ^udEKVL1UxarCN9 z8XRW;bN0~6TF~3Y`gEs~SaFg7TDapgLGuuv&bVryPxiGRUCTW`ZTl%Wijr6*vRVB5 zbR9^VH6q~(2-F#zIzdRGr^l^v)Yc0~#yg^9^rDis_5rxApr9t)6XbJjMYig8c3FU% zA5V_|h^>i*D8M%shx9QN72VxHQG;PW2?f{jqNOnuR;i?R?mH0;$5ORwnuQn2qu3Yd zvnG&)_xAQ8t9x#(%i#F~^lJHIrXw(;6PMnBUGS9n30BM&l^eiNm+EiN7ITogBIZHe1_47uL!kDL?G0XaU}y-*bDr-u)rSiYP<4;) zy;OVJ?3A|W?FBvd~V6U$m#TkGq&k@*%CwF5(fC2$kgaMN6XsYs>^ zwKVPJfgB?QP7a{1b~#o?Ru<6tSyWV7a`N!dkRbGuGouINkW7GpMcEnV4MGtX4nalz z()PDAU$qe-jb`~2H|gj2cliJlVnKqRJds}32Xr~c$~w^YhF@bR@#6Uuo!pIDR9k5O z>I?|~S4~7({lKbik{JveUe-4tgTU+2>cwBQ8*nlh`|FZ?)2;@0nDqOkN$anFQHIbH+iwi!)_fnHq>f~rG9egsX$$I%V@xUG4 zWVDbdYAY*K?%qAsv9{`ng0lRmj-n5$6UY{DsOt{cLy5($!CR}k&rXp>f$kswg78}~ zPruC5zI^@A+A6@na4Yi5^)_lf&9_d!U=suK(^TF;o4M~v#=@T%Lxd6(bWuxd7Lf`p znsd_rtk;O(5cZ-Ji@eaMzZ(@*Bv|6Yvf8=1x6 zAKl{rRn5CylkR&Ut-`l&8+HARO;;P3HgDV`9ILG6#hbZl^nDaVNnS4F>W=;Hq-Mt3 zzTFW!^C?Oq^y2N=&YFN<2?54WqebRq=Fj*{WxXP2crXkCc zwCt^}>i6xe#{tSU{wsQ4;&-bB#eV&urcGPN?yIwV&q7mblJMJ<)H<82fF!q=qzUVD zYJ1|pF~5vExU;0B_>&S1jwP*1Aq-{%l~Mj(JLawu+10yDwi-X4?1P4%+6&5<;{0X%9vyoupHG!do%&(u_n*(p{khWm-{hR`+|t0q>!4qLURBLkYrBS} z{U*ZSd3^p4S6$amz2gt%?N_$oNTF4yo=Bw5-nsr`kH`PvUH|P~c<m0GynZi=tOX2QI7!O#&bys2OiV|L9tS2S5$^um&^_ZvWBbjxg%stTowRkI_Y&SU z;fZ&>&JZVi#ZLGZuUZGqi|?NQ>#F^I*7d(peEc8Y#e0AHdbzN&RL0R@!scqOw`W9| z)rZTbWM{2kx$z{)k4rc1ei~yF@JW(UviO;l^6>X%C!YQHtNzuu5}68(2>qMeDkA>< ztn0TG%_tvg3#paBU->DK_;&-N4gThJC+a^lO-b)5f1yLzAp?7wv+s#j8N-q$Co%;C zUVL!tHc&dnMnn`nQy(V~tYu*~`Z~koTy~-*Mc(`A9S_I!bPx zY>{=4BTjnmn8f=>=H;=f-x-6Rpx-3=6yY|EZXMLTxPD(m(d*Y+|8C;JziiY$-~7LM zm+8L2?#Iq?zE@O^OQm=vzh3Wt5pDRN-RbXh(aH7y?7i%K-njwiL(O^qB4Mw^En?2w z9?$+-|Lym>D}6t7?attSnm1`Zj7i4`cb_8|doj-CY-kmkstR4Zdb6nj;pv45?|Q`^ zK>Sx#N0haGEY_E%sP12$b^XUr;{L->mV#&m2$42<+sJ${JQs!F= zm#^+2@}R`=kN>KrCG?8DQ;Ea;z$U8`B4;ko1%*qh5uT+<{T#2KU5#{L>n-z&lJ7$_ z9Ot$ltuORdycVaG!P<24JX5dfJ~jI^(wEn2)i$$`6S2y}S!rH8wDoTT5DwBmwMWg; z;#1Zi?B?^oJJ+!Rt}o~G`j6L-`nbn`Gv@!_EdM`mmIz5&obVqnv!8w?W83b8_`T)U z0lAvm7FQ+~CAFjG96H%eyS^N5^}4eX^Hua7mMY_xN@4 z{7>)n_hsq+x82L%t~Y3UUAJO!{`IyRPLR??A8u6*@{VdEIO8npzaHrKvHM>*+yuwv z!9qX;u3LNFRjfkH)#6*hpYadzL$hqm#xwkMLe6>wuhcs%)3JO@^I@DS%86KuM|pH zojefj)acyQx7Mq+SPQe0ns{mb%5+(1Jr>|$uu+kKSsJxuEm$rO|Aszz;@V^X6m0)? zUjfVf6Mpz7xAz-2bF)MI0?}wr4tfv8*#aLu`q|x`dHFM0>H#ctEicC}$un4f^?u~B z(Br+(WbZ3rsI5RIaTsO2@^YWyyZB(610?=Ik0EUUyl(4d4%*7omkUBzE!9bUiP#2tiddZ?5B+shC-t04GBf4>UXW*3B$j+6C%a^gr7zSN&HSY#EtCi z?2u{^?(!cGqK33L>E+9x1$*#`^1nMK-yJZ(r6s_IUmqvjx?sWYzFiOhfjp1(m0Cot z2sFa=HgqQ|LVls6rFD#o6bzV%u<#Ss(~Okcygu|jiMILu%w^H&oFoD9qr<;RD+&#l zM#jcp8ylg5wP{1^>Uk9TP!Rn!GZW;N939;Wf)5ngmMvRA&e>hRZnn^uO$|183H@!z zk3idkatLDmi9#ZwqsVUXabcmWl~roefy5oZfo{Ntx($kf1Q6QToWu(BXH>zyGT~L8l!&bsR(VF^-R;F z-{l#^4Q zB5U{wgN=dpsAYmyx!%IqxoO>SYU?&&rI-!@ZKhO$&HeWp#KpG%l}6Z+|J#kQ3@dM* zfhd}X=n$&>f#Koe8#BJKto;1B=rGmTBSn3&6OF@Ywgs^Y1<}0+59DQKp(2G| zQtS04bpE_Yr4aq5)1dyrQAI~bqyL6dg33Bb4n-ssLD1L%W}eXYR$ym_?viJY;FvCi zpE9^|CGPoiw97cfpOdHkbDY~9iRQ>WDA9(vjX@$NfM8OrjC_1ejg8Y4L2T%C0v+cD zaTH2!ux>u61SO<#X{p0?C|aSb0%YKz{}mGxZ*On(Z$N>#C|{am-Qk8dJlUu0207FG ze0;P}c!6k!8OnU_0P)f@Jd84t4tf|Mk6c@wD+Y^loQ;jZwK+M>L+(Ccw18qEnuNfF z4!5KqVTgkywlPs@c6Qb?0BZFwks?ctFIA?KQtgMoIy*W2C~^%eqWa#_a-W6|v=DTH zit6dPLg>aWLy-myi933(&`k!$9r`7l)TB!!YMb6r!u^(c1Z|Gy zg%VnM7e92!laY~8r|70LemAWRMMwO&Vzj;% zEOdSzCvF-dKoIbD7Fd0V^li}|i7pF}wYw!YZM+4wDcCLA7PZ%E++U)^igA|&X{UY> zh&sslA+kB_>FMQV*_y$`%zU@K&8ZKUNkEv9blTk2oZ^Qe>CJ` zKri)&=H}q@2l=gjU@2S3Pl0&*Z{4@Y@=Rjz8Im$#Cm#nY-V(or`YLFuxR%yd`6$s9 z^t4W&^aXbZN)qZ!bWhpcalLu-?vDKuB>T~VAuj$7#l8XZp}|2yt23$=lzdO0KCQ2$ zFojMJj6DsP;U*!k<)xgov)W7?{QPsx`tG#&8Et;_&UK{@1eI7X+Edry@sBJW0inq4 z*R|<)dQ#@WLUX%*AWRn`&b;3bCMPF1@bYSNs&0rR&IlUqp}TRY#vZ4FY3!>C!9Ah? zqTyIFp#2M<{w>qw0t-t5bckM_p36`>ViQoRf|%g>UQx(Fu&TZN{S9x35t^X-l-@e` zSHQ)@JmeVw{WdIj#Y#qolz|bQZDH0$1_UVxgh`MVsjI4fe5g3F-apFS*^U5$K!f&Y z1Zxjs007Xc1B+~hAas29-{P7ha?+naS)zR z;j!t`ig;P6DkzXK#DOgK3;>JWo_D>ZsfiDkE-G3S9eoLuIm8i+o|`spLKkr9=g&)v zi!xQSP1k9Sb7evyFmWy^;z6GQgD1MMlHZ;`=1CZ>;IYFgmVwv1uux=$f)mh+hK+&2 z096V^ZXDZg9JD;EhZSg;l?y)wy%ZM*$LMM?iW=A31RwCBjOH5)@BZuw`kT<&wlqKg zI+LIV{`tWR^*6XRLgVHA`>Ju*P&M)4>@kW=r~K)Zu6Yy72G3HfP5ieNxA`BfxG2(~ zIh2;qoJwTw1y`;WBKxs=34tCwgtzxO2w2i|@(({Mf?5(O zlK2+kI*QOcZmN6ew26kql^Xxk*o@{;|XuQf=Kk&@K}-yF8N@YIRj%4eovExfu$J6LfPrvH9SBj-o^>}Ff2h@hK!qOVMHF5Y z6co_N$SV&e)SWve&_d%>;q=|Sd6Q5$1I0~HybRRc8UD(rp`#NHr-+Y6k(ztnT}M~< zcPGduSB<@_CcrqTwuxTr6o=N8(BTE0IBJ(r1)?a6o)EMu5cK_U2ZZibY3UYN87j0} z$PP-AEI_~qfg^?qLj$);dKae8{H3=9a(Sx_0V{V z<^i}0MD)#3Et}p^)6hUAHHi`$p47IDx|SDvLm2Y{<*})Sg=|8&kRzd)8V@1hICe1C z(gHpu8VYuV;LP2yt3tT45bck5NSnTX{Zv|tz+^wtavl1F=~a(+23v&gV|YkAd;8s#l&C!>q@{8GAGT_VF7)hna6uo= zdRvp)51)5DzxBPgZKUUyuLf!To`i6Sn3k51425IfNRC+p0-cB84m5obgpyCtJ1tQL zWQ(Qm1_2@gn&7F~(`?-*~b|dP*&;WD}4hmR&WtX@I z5k_XFE!^IKnLy#9xQ50Ynj#*I#byB8BiN#C@Cn&Ysy%x~+ujg{BiPwpD)rq16*SP8 z(WEuT0*I|GKsjoG`nEaxU%R_`fY+Zr+lK)r1;^1n3}8ZNXvO3PxN^MuYoh!gJTS7V z&&ke)RZqZm!d)M3i^SqXje2!wax1j)@B#D(4zN*>m|IwMWOYS*{mMl5{BL4o{{WNU zJu;7a#0v*>7(y32x{yhED*YcbWbU@NJ;1W&qU;OojO9S*1189vq@jUG1)_mB2?;Ej ztubhd#R(das#}+IcRyy5ZK2;qvI)o!QhKez8`=5!0t|6Lg(!;y@m!un4hg2D0L%|r zesviCI3x@hMe&U5@|Fk!X8`ly4$CqqxeKSxpM56Ueh{$~b?H1SFGe?PN7l~*0-tx& zGiY=s%x44 zEh$kmIVXG3tW8xizFMnP?XC=M^x-b?yN3@QsXFmw=Xp2x9~~?mH$RZF@o&SVG*PSS zLV>wG(z0Vh{o%4`8yz6qm_fk9sP04J2NX#C4Irrzo)n3WnA1cjp8~@@3;3^46 zz-;MWedHt*5;#lf)6~}E+)qP;WyF4Ybi~HQoK_><#K+ICasV3&4G;qB3p!>drtIt% zRN$2%;J$H(gPel$1mOM|-aQ_S~mmUtMhY@#7%$ zlNDkvJ$Q_DX2wa}P&3z}$$S;7irL-rl0(EfN7mk$mN{f1)&{NyYgdH3$wWbYN^ zUFh3^1|oKD2T_|Rp};H7cEW+E0Mpde0fw744D5t!@VS3~8CuGhX=B?MbdK}0vB^$6 zLt_zY^@tLHcg%3IP}+^o30!(d1j#y9{KX6A_p^}Oz!?zQP|5-raGhN@)C~>0kQR@Q zj7XBF#l-v?Oo)mUv>$@nB!OD~Y7-ZjeBDF+(g=}V2et{A8G#rfc%XV?#$~{8jOW5jVicd~ z89-1pL9~dID#y&j#)kPgv4lZbgAEXE!Mi$KyQYZ-F6}wa*I8L_-PcxemSKmOW2362 z6&f6z>$>D%o(3rnoG4uFIDCeYQ4l2aNT0y}p+S3>1RXlau`_USM~@vtcv1*E`K6=7 z9cp;=yp$$w56s!Kx;jpf*uJDM6 zH|jcBmP#-d-tx41Z7;u{lmnTDlCvsSyTjI+0gc{MM9r3r?r=TFjfGaHRmry>x4xpt>IB38v$IPk7p#&9QKfgP- zZxbdU40SET8exyQFdR^2Dk>QF_iRQVpfv0CYef~6=XVQ9Dr#!hkeGjdaNx6HIu;i_ z-SnWfkR||RjiZecs2|50^D3VSJ7+*1osnU4_3Eq6hVHH|%wRxE5B3QHw|vX*CwG-V zKO=JNSW-$MJGBM&AEB)6wZRz(pu7tKPnH%IphMI9n7KwyhuzB(Z+o#CctJEqh(tOa z^HbPZS?{CecPDp{vDr0U>wxd zN$uT`xnh+Z_hDMx0$4Cml+Q$61sQt0>D^3cUc)RfH3e>X(&;TSaQSB^euDI(b*S$C zWnA^_*Zi-`N)M?#(+L%?ly|f{W0v4waMtvSZk~6Q*i00_Dv=vFJvsReI2$oCK~TV9 zmS1KI?!~)p_k>!*2#{ zpf#@>fGtbV^daF?Ptjt^s(ki5TiQs`B zfY!tlBwxc?83K`}ZqxKqeLeIjVNDFDc}z!rmqM z@GA+*F$9=ZOA6d4A#HEZ*LGUG0L|UI3EbrcM+zd3$Xpm5nfDp%yKoFpX`#=17ZN4^ zgp&Zt(7ei$oR^ck|M20-6DLds7CmY(jw}&LHo6Bqd+BUIi+{csBG0!p)$867C*TD3mXEL#%iqLa@|069g4z)LlnG0G+5vp3n@h z908NTnf5L<1ZTR3ii$v>f}oTzL0T4$1BdY#PmZc=a@@0b?>PSKi!sl>`?$}^@6N*Q zfMFo2x9-^YuGa>Ka6bjbAr=-W3A13U$iesT+ozaYc!fIzid2hadNS|b)YP+GrQ4CH z<6#aQ2=nkDGT;0jo*RR*Jii!YY{V)uazIKbfK!r^pexk{sp`Fb9PeAi*Xz8!aQ z)1S!1`7sO!v@t=Dio!sn2NjXWQ?LoA(C+}@uCwz3q9}&@Jqrm53JzXcTJpYo7Z;K` zbLKL%eE^|=lXxEiih}X`Jr!)_5h0*8RG2tc8j(1Jd4Xh_lI z$cg3+I)6tL4)7e4#AucR=|;$Sva+6Zl>NL$=sUs3L(QmcJ0Hxczlu_KgUM|$KWL1{zUPiV_ZL0v2@gpqL!j8z*mj2IzyZ+B zo-&N!OxUTD^YWZ9<>7E76%Rovpp^igDxJ926%iaPGVpJl5DAg=amXRT(qTXv76J)W z9id4v_W5&L%#ws7Jr1d?;3o`oMH&Xa1*NFw`g(#$d7=3`VH6ffWpLE;A_ut)Kcews zJ~x}tparpLb(Gizc|(&E`{3X(5T$GDKyz9YS`lh>868Kzp~Ec*7Oxb43%-ERiI{i& z4uzq)Ig;o^)Q-@Thw*&y5fG57ivjuoRiN{@EB9(!0iTEnfq3<#faV=Hjv^My3vfE- zdLn`4Z~38K7t>)PS{ZhXXw>=>7J|UtuezST;i86$rL9&fRj~~9jG(FxCRX~-1j!HsX4qzM{ zh^k+BI4U#QNW~a>!1ZA785kHbw-;U+fyqINbm$ndCxD|>D7u>^!R2;61opq88V z@jPaIl-@Ax1!LQ)*7}^W+c*x(7)~K{li~IX+D{kHbFtzKsbQa%7lJ`?R$)Pd3jOD7P4u@+bhNIQ8^1)H^L_~i$rg<@mg4DD2{fEy1mEyOQ`F>Z{7rZ`9G?A)3}`1F6{rJ zsFZ||AkSG8IW0kg1RT=n z-uK?meZP2K{NMbq&;7agq^{p^p6fi;v5s{tjuQV1G*|84y<4kz04%7oZ#$)?SFT>A zuiLi5totteNs~;xmnQ%hAX#vM?LU0@aD9Cg;BjQt;k2FCJ9ZAj!KUxI;MoD-Win8= zb1|AVs#VW~F%=er-@6b5=6PEeT1Ub`E!_vO>4T?<8gc=pVO*n6vSf3Ryuk%*Lq?b|1Mw}IED zS9%lYC)cEps=bk!nJKDvF)`+xX$g}?j$D7;X*jes7CzUmb*Jv6t-U#G4*p#fHkbO# zO|i3EIb=WmYg=3ln#D@7;?Y={si7*VK^^kFXw6MsFY!C}Y{d}&7UyTY1Gdoy5xuc# zDVH>4?YC-b%JRwYAEl?`kDQT_K^Y>4Ay+q^2I*=-k!eG zH~1)Xu({e!Y2`0HtY3fAQ{>fH-=@rZ>()Ese4&S*L>+ie+@*>MH+kGpG0+aj!^fk2 z=s2N5${&TGi`fGMzen$Sd!=iHQsTU9yYDnqIVMak9JqnkK-bQJ;of2CUwKTlWUi^N zOMAR=8T=Nzzh2mHQADAL(5-lZG@aJ|oX;E4(FqE>FM8}Th1>Hh=jKhS`j0r4VIOLJ zXcdXR(B|XVk2VuqHXA9wh$2CK4~!#%_wRpL)h~AQore!CPCmG=%=V+)b`frvuyRYx^o+Q~m~IXas@YR-7uZO2~(Y=eELIx?er3Y^+J!kx37 zscDV?I3GWL^MWC4vR7AY1NIZNmIwM6j`VjE)J~~C)XsWJ4&nRvucfXEoXID<%>Y+W zu?UCJ1H#c|byk@?C}p+GyuY*n)63eX z4g6zYXWg|H7P*`W^yZE19Q5k->&3il+VAUHLt|6^Nuke?{zCR;n)lDMM~=*)?W3@q z9<#^PK*=L=Z`|lW&Mqq3OU}l}zenAGBRo|>BLWT79;Nbo_wU0I3Zph?TdLBecn)Wr zD1(RwRmf7F0E&uRBQVWgj2~Arl@Sm#ls*L+$|-w(VdhjuD-y*c7;?%tlx$hZKwD8O z)O^nRpb>vfO~npCBnk+?6wRk};~&_)``F>bPiVxYUz6P8tj`=^AEhvzUcHbEyc*oS zM-Nc~hO@y;(ot89KSKvoK7}4kqD!l?@7(E+HzME59%-SZ#EA*%f%mh(VBji*731B0 zVjsr2I#^%7w78fn6&`Nxa~3Xy^UHLv3vZ)JVnn@T=guQBF>5H*78cre-$j;u7G&Jg zG}MA_m*;G^m@cL+E+0|4P`w$5OSY-}`b;>GT(&7}Eg#Z$3^SU%9+?eI6wCU>1CK|0Yo!|&52a3d_O&e$psB}F&z1K^6z-MBPk5UI7yzf{3N*gw= zVpGSC?a{k8sK%V$;jv=_ru}ic@4pC^SzBPg-f6r?kyl0oQn_A;XTn>8z?eUxyKZa*g!3awzjh~ z931JGCsAMF=GNvHB{VinzJc>|Epme(`1lTVJi^`czE{3#tkSZ6Ttp<85M7wb{@KR+ zlVHcy)h~RR0gsM(!`UwcjE{2RftR|H9RM6V72lh6XTjN&Lnm)?dDLFKVhunchBVQ3 zu8jQ&1Oma8oH@{;+)Owv^I5va7)B&D)iY|GjO=S^LcMXCha|3*>BM~qfHX&~Xw}Pj z?_2f_<~OcUex0wqa53Dk*`hBm6B+{RkiG9TS~*eZ18co`lV~Lbj0@F2pHzoowV9t6 z7J2NLYRi^)s-{hr9J#RKi9gO2^*2Wb3poh>dv}S7XsB4Z(!989uSC{00psyR{cc7^ zYk5?KXOs^RQ24DUkM2=FzAQ&6-y8b^n4uBGd_0Fwhi=@BZSD5JK*z)q;;%U04j*O> zV4DmsqAb~F|M|QpD+tka{vVEun{H<}4^th|&7`&wJ+qUp&QBC634ahC?(>Z;vIYyC zonaojV+WS$2ws!#cUtZUeyqYL1@uE-@{eyv!jT-MV=oSC-R|I)hJypOqdd2L6a>8z zvXYhErl?V`K`S0-D75Y_+q5YaMUK;@#iuiKb5C5n_>h7tj^_Nz!qB5!IUwTvxpQyu zWb=Fq4~UJz%DP&ylw(Klo;xj>3#7aH>|lp(`5DxNDDb~iEy%lm-F(7?lcan$U#oWw zW0fKQTjIO#`q>7W?zNS=U*}gWp~sI`kVq5bh{dUJdSwlat>*i)>_? zL(gNO&&CAx22w^!c9ZM;vB7nALeJn%N==VKz!qMPRK*kws+syNsaD#tW6!CE(B*16 zaixdHBMhSK?AmCK;%>o>sop>Jn;w)H>zFiT61;V#rZTYT_QG}Q(j^~Aj?tr=l-Wo0 z0I;c;QilekJ_0Vf>^_I97Lk~Go4n?cPIn7P!En=fq*&5OBA=NmP|VUvbRicOj&hTD z?!oMWp7r+>ObLWdn>4wjF6XI~{L~n~0B@Os8#cet6ur!da8Hx12*?>&usuduiYRs3HIh~Gxi;s83l%(PVs&L@41_O6#w-r`Vcq)o zUx}RuY4L_&ju`jVTg9cIaIrnz4V4B&A~F0lU$Jhlti-$XHEC2(P;70gQvxAM>D~t= z#^mha_0MnKzKzsVxqkLPQ4nho3Y(PaC&zPLBV;(7eZre@e(OLN3NnCCHt0zubPBb; zjXc}Pj2iRxee859CE();1cV+!^Ycuz+RLVzee(7@u&5)xtuBT0dTVgz2hS?>@3nv0qj+tl5mp;!;{l=X=p zw18J1`bp)+CZq{j31|qzRN<4AnQ4+cLbk$YwNYoy4VjKf$#?3?50(bkCf1ZLH||vT z{xfUHFPI;Hg9*?f-@27nBrRKS1(;B&v|{2V>Bsh@l8>`zOVCZ(rD>1);Zv2HFj`F{F~7B6mX zD;IHpNomsc{QPPh?1k2ct<(A(-0RIq!zO-HKdd&Yn*QOee(9qJHXV2RwvE{&jcVcF zUMuHQpWC%*(=`AZ`|BbOys3$PW;7Rzay?a942;2Bw2U8mH%cNI)-+DYE|V1W6eCg) zB??DH_J|RXm-<%4b{TaxUMEr%Cnv4-nNZ6Z*GCz)^b4&H$V*Aod^)@AVQ_V#*&u1* zs`Qu;KKJU|H_}i(r%%)M3B(b2B%vn2~m*n7Sm ztf3q#I_C1blUz%6pD?g{*mg~tHf5%ZsDivB|IYU?|EewqTY1`%2$676L_kkjW^cR) z{tfXe=@=qZB2B)!oKu&PAKzWr#2YG2Z`H0xjL+@gJCbvilexdU#4{YC8)&!2XqK_D zDaAXWOell(Au1= z&CDJmUq+7JUhR|4!M~sA`YK7_t^)_kc$*IP_CrxFfo|`rgWZ3qG?4~Jvw^V&8FLcI zg1ob)ICy_15#zwcne;t%3>TMg{h}tmPD{AaYDlxx@EuN%!>3C;pS*9*^?R|4uWrqB ztVM<;_-I5tWLUiNnrG$*@e)k^L;mQ`6s@05j-w?X_K%KjNccP3${E*^=>sF3oSc%# zQi!Vs>T<@#RO~CaOP*efzx_1AuoIhN9SEMv2?MJE4l2+2$X2&(Yo3?i`$&pU1?Uo#1V z^5)H&QL*~^<;%6aL5=*{{_7%KGp=6Upt3VOe7uKB1wgX24JC$`c^xSL6V!MXk{_M^ z^%e~rmo2jtcGHK4%GJzPYWj@3M6M4kSmEMwx$8#O_A@CMtB}OXXef}A;Hbq!Ap1Uh zNH>Uzm6ineK&7qU+x@*d)zyTtudL$bMsL-WuJPi8xdN&ZY$1Qrkl@;*vEL=rrq^n> zy!tD!C|eaZ+Nv&5NMISmdr@fOkgK66?V3&8X`dOqt{Kb9IGzeV?Fvk8AQm1rRLrPv zr>_3?{d=^lX)B&w79=_|rQVIo@2#d}UVEZ?-u2R5lX zb8U2b_x3^rFkExpb0p=5hOU}VF_*>61~G5$(y09(``Sgf3T;-jZdO+Pq@>@`WKW`5 zFDczt5sC_pm%V6FFRCAPLD*=3cxbVQ<`gIntHksx-cZpOf{JDI=I8 zK;Q-Z_m~it1UJ!d?|6@%l3jjI&HK4J?{%+js+AWtfzyGxWU6lq8l2!PM7Yd0dHs6S3v1jZuWQB3l;Zd{z4QbdV z{kend3!I(FOuW8T&n6U7oI-raxUSTZjr_I-uj^m`bm$h_ky5xcm(;>~U8WqLz zyuxkk!7*PI8>C(#)yS`A$3Jh=Sc}|TI^_Tw%5^(xcrtW?2IVFGpM&2G*<<@}s!;yJ z6s7HHI~YkNw;@>UZGIwb^X3~sZ3m`2v5%=~QZa;zB>g%%Mk482RpeGT`Vpu8@0c@c z&eEk5cw&p@FIloyIJ|AxK-U1^8xDH`y%5nYC4N2s^<-5P?9}nS;_xy~S&M>?IW*@f zB1J_BCzx0K7>q-DeO>sw8Eg~16IT$vYG_l0fH#~xc?^a1gDtys5&qaS(mX>{2B1je zQSesIQLT1;KCnx2lnc(4M3N+waaPAqxDi{)Nxlu)xn ze$bck00l8ryo1ko#sZeGPN!H|8G8(r1o!z@ObTqK*xKLXN5#<~I0_WJlx*!NBC!0) zvl$M=!GBkssDD*X`eY|fDr(ecm}I~rDuh(DZ|3EdB`@Baknn+ZKXb+m>_B}f?CXb5 z@Gy`>>{EI;!T9eG9M~+SuVf#iMG~pL0TqP2ld3N&7yD%A5Y}dTdOfKg*=~ihyrkw| zE8W=k+|ouhEzuNnXk1$L3K)0`-4)63YB*zxlpWO67!@bDL{~Xk3a76-Tb&KEbiK?$M8b;D3UR|&PW@6a#tbG5i2gzN z2hJ5uCkh;x5QX=jyQ!fdS=MM?fYYpqagN%ya-Jc5yyJorWZoM83A~ADz2ZS*iD@87 z6S;zLl+o74!w_rsDbuHWQLv>80uNZq`sh0uDEH^h7X=4YC6ZB*#vZIA$)W*kzpe0p z&{k4A1^5ql$A?-0%Yx;Mx5_Fc!Q5+WQzjFzcqu_sd7}fZQdv1Kd`P zynMTB(J`;*0y~Gkc=-~7dX1MCT+H1AO^i*24GQuHK3Q35zV!#4lg^&Sb$X?X3uT^% z@LLrVdXCmw6>&!P_~l&xS>49Zt?Qo{q2BQDYpD;dHkxH`-=4Uxr5Ovj=q*|n^>8&aeGcZn54ciz%`0< zOJxKo0Fn}{-n`S0U;v}_GwzeMu{9_NH8>yq7>|^~KfQhX79I|Zm%%R2bD(B>Yik#D zOdg3y5V>f|>C>SD=8aqa=!D6CzDR#l-j)E{^aa)1`KF5fyja2UZ{NDL=F_L6hKhbk zq<+HQOgz}K3vkuf>PN*!MCj7(g{p(Epc)d?aj@l-DSOMKPipt+;)?;U`qOF3(#wz3Zosr8w_jmvMnPHt;_c`8oDQZ+0QaHEZCdi6$6 zpe_^ZJ75ikaTk-MmQQI;>WRz!)IO8rylBbkVG;Tmr5rY8g9at{tI_!zc61y6sA}#T zW5Z*H!`L}}hu2n9F&d>Gj(lQ*M_a#36cPht+ktbfI?`rP>(#XyX&DNDV21(ZK~S<# z&)B35)YYws@#==Ow3Lx({LqVjb&>ug{#&-({vk01fm=#_Z_Wb9v7#tM0E|5Pu%!w76K@R1Cq6JO zh3Q=Pm%yKN)X?J6QvAeAAj1*Hs4Exm$=%YtVNH>r-ZB5|)%5*pcD1EqixvdD zIyhl;^rHw|6~-%9en1XH6Rmxp$-GyZ7}$Tf^KQw~?%Nml--WNvv(){w8do_GH)+hH zEFo4G_Z%I~+fPSNgU|^&?}!{@ReAOY2FznPT+q&))^XWpFPp^59yfK1m~y=!<%gPK zzH+5y3!?EBPI$_=98=8KQx1WZ z=^Q7hKA#)^_agdNK!|7(luhgK>;d~wSFm3sE14L_Fwm&YO}IS502fCYnZM!x`S35% zIg(w)$HY{Av`4m%J>mmc0|wk#_f@;o*MLafqCDWpuD>Ae%qK5ih|LInz#F}vt0{2U~qq+)k?`0L6_)_=*99Yi4y509@X zD?fimUeIQwzTd&i{|@X8NOH4OQ@B&A3Y0|HK+Ima;Ki9dU9Kn8`MGmeS7h0zq$5{D zl_RoqazpFOhmVi{)%0p3DN}JuaNv|PkXBFv;rUf=myrx4tD#s(9nJKuQq#Ert#avy z%E~tfS&3Pbh8xHMV{^6A`SmkQ)vUVrag)G$BjFOW)S&|J^J~Tcm3DTwX{<`=UYEqNU2UB zhxkHarO(vU88>e(2EWiFLCz$pXjI@m0yrt@0L6kv(bSCeV{!+X3PYg$K8xe4Z*}`N z@Az}#WLit}#tz+Q9{bgM<;tBbiQzZ+Dc6?kGeHd6Iy$#=aw5cS1r?8zk?dTb{cO_x zwEv<3POWlx@4n&T1&<&hMwCd(8mgN28NuAj6e3tuohA8O)RdYIN{=|X?aS9Mg`1uq z8XY^kpGe^EBjA2rQBgy4EHcZc!{Hl_%{F^p=@tq>cVp>bUGfb+yI@(Ix!Xe|^_oLy#1e2ix(} z#)irPbOUEQ>QZfc@!ZN?&hOyW;G9LdtwLi5cQt=}^O)kuO_OB>wW2!wDqrfl1>qWs zC^#~pH5?iLE`M~x+if@Kyaw&B4PRv1wBg4Z7@4vc&z_Bx|8w)^Bj|sOA7xc6a!uE` z-c_b|<3O_93G3d88?t5!OaX8VLYA5d$*f3kE4Da2E>-fI)lkEcLA%cDq-sO@Xr5-! z!J(%4^$~w%V{#UZc5-y#zH)m0D~U(T{bgj zFgk>@ND|ycX_J=EjJ+AQt<=o-{FQ5KO5bZuD47jqq(xa`Htc#QU7d9?TDm*?WJ*tqpDJ?!Jrd8Y zh>j4r3LD4oucW7xq(<%Echjo7py&JsUY75~Uk=*NoOzNSL}py>-P@x_fVL6u-AOt0 z_boWiUsrMLhj-va;X^cx!bl!n%vp2F1TiVpHpBenx*q-MQPfwuENV_u-)?Oxv{@)J z`AnKE3w>nVf*N6aj2fwmfo*^oWh*7do6TSPtx>S(wr+>ym~z7t(q)k*nv%U+tzFB0HvWu)()@R%3+6 z`3o1&{0$?Yz>Pp0p}np+>Q-#m7_?%hshZ}jzq9}{`4bxW+Y4%U2p+?5gcx*NGXmv4A%qKk!}dulah$^r&3 z1I3ZnV?N@+T9s7D_S-vs^5kpt6OpEp{a(nI3Ai#lv81f>aLQ~^%JCZlz@ZzT@zh_a zBa4S2(E$0N)DX2~2n>qnmD~TQgoJ6SqT)z$L51$r=jwl|=;Ib7epZEoBU{-86U6yW zVVHhB3ZR1fHGV`bn-S4?TsSvxz770GqEU)xDVd||tDUY;hJkR_w{O39X+Z-#6g{}R z^1RisrqV}o^(rNKlu|T-AheC*@R8Kaq%{`h5c0bMbXXg(RoN5}P|qY05(n3@yIcz= zZY}TZ*+R;9KW!M`&UcOzYo+1>GNk#~Ko=juc;)5GO%L9ESt;F%fkCJtPo7)~@k7VF z*ai_nFor{EQs4}4(h;!R_YMLiNCd}~>$(ba-9d}K~Cyp1M zup|)IA#|khD{X9c(MGHJ9I-~>k0A&XC@&VOuYu+gOa0l0A)Rgs$?4lK>?T&kyu=ce zfb6@j8&uo16T=Ue=wBcXbnR*iU(Tm%SO2{E;1fAAu}zyu5Ai{#I|SNcZ+g#(lP2Av zPO8K;zcT%Yby&PK@no`BwZ>E~t%W;CL#nC6hAt)baSIsrU(B_dU51AqZTh8M@(2D1xJ8+i~<*BQ+Qx_7aR6*K`+uRg3M zjl$R!^#w)PSx9G~2!pccCYE&ZuGvAp2=x+_y?DtI9CxlWiO~RSn-!~8v3!qqh6ucH z{yg;;n-e4exsVh`gN~!XebZ=V$aw zf5DiK0|OVeO7>G6pxOLYXTmIewSK%o`LU*K>1g zUxqK2M-{1wj*PFZl_8$D>f9Oof`eSh%yTP{2wuy}o3Q&Ju+>V@ZoSG?M_pDho&dL& z_t(`8r8gA8F_Vr-erTR<+jm+GNE)_W{sVUTO*K?Z0crf(f!;q{s}MFHp%&`w9Gl;s z8GYaMT1vZR$ohGM{(U#7=!f^66T2sV;NJ>hC%Tfxkh-$)klgm$djZJ=DlddJ&4$w^ zb!^_0R#NyRSxJ^=e{RKWj@5+=drHH}ZJ}I7$3ksdu6c(fb_X?S!y!Xp+fUHgQ+*md zL+xMGE0O#O2OKZMkZs$VYd&Y;u#9;sq^__>Y&pc*@FQnWpN>%0>))TlDf-~SSsXtr zR#c-^pUSj`Co491GGC5Dr>e7Rs#OmmP5(1A_C>#{rh-v0Y+}T8U8)~M75Ia5=R(6G zH2X?3sEAA?3>#KI^lv}^k7qSjE9-(QWwZH%0U6S|tss}Mb zt0Fp$Hv~iR5kp)Ff}~yqYns&H&?1xO-w+TrO3#&izq z;i;`zc>5~ZOPU`U7|j@%BU@jQVbvOVmbH?V9)TV33aCa#$e}TU4om67Z4m>aoG^hg z-R+jpvVk;j75cHq9+jPxUtxPa!OV>04T*Tg$M&O7^ud~_wJCLmA2by+Zj)XPwjxYL z+bx5Ti928KY??!p_vI^Bn3vt%B2>dQ`SzTI)J|uv*!K0nV!4$mDe*M#rx7OWg_$O-XXK?VHe|{G0_V?1eofoZRHUm`B$h@ zZ_u+M68U2T(uedDCxq^{HjO8jiW0=_=O$&J?XJO(&dgIRYtM1>qbFtuMC^;ysyNF| z-U`OhxMC#_1VtXTZ4aIiU=rtb252tCmxB35ra6=(QW(n!JfawoOT*Xn=@(9)UP0UC zwr$(!G7&Kl!5atpgQ|x>)`nqwlhHYvnVrLGf(j@4YH|};C&*|8P0?z)WC@*vW)>D} zzrLG^UIpxr+eZtMd(#|PPShf-Dj+N(Ep8?(H1r&`1RiwL^&gmTjn3=X@#6#b<q_hShjHCHxLnZr4WkYq);diP|SyOR+F6~ zlxBx~;;9q=OEG;^Jt1}-W%hP(xNb}jE#_yG{MfbOW> z5Nt4(mu)t7tZKinyXFr4JN3>0MUubIo*{rRAE0gT(-deu= zVv`IHg)D|C8#Bia&H(n%3t< z7s;U@!RlX8sS6LNJ;g)@1Nu<-sT$+QF;BOJYrxKY+ou6ioL@t&gEYkE#VQ zHRbfw6X0!U?@TI`sN0ZI6{wFhwfVD4!&VlfxSl zKdsT|k0?X|7h!OLzrTM{zTqdE0a(Df3@{Tz3#55*eu`Ad;wMYP568B!YqPLON=!T| z)qJ>xIkGGNS_L_!(vxp(w|8#kRVU#8_!aCrg$M z?iVb{%3B4y@E5{bX9TJ>kEmT_A-tPksF;NWluC2V80COfjFvZ*-+S|9ef-Op^AEpr zIX@$F#EVYKQHJKFN7xrU@DwAjcG5tvVNb8xTq@5r$N$dkj%7e~x-j+H@{ZorKKJnJ zpivs9k+6^Hkd_seJrbek+A?d0$xZf&^FMTClL17M;K-M`m)rbuoeYD8BoHWf!%>$2 zZ@~G90+>a}2N{Na{UFGG04nD-4a^FkgAX05r{#V-lbW*T(A#+-HFY@MUgBiDb*r5C zLs3m$^MCXoJGS7fATS%+V+f9KR+KvOsHh*(H-fzQ9#b@1I>c~xFz6aN-srlLq-`SU zY1Mt`Q2SZ4gwH?lOuY0z$ILv!z`r}{#}<&SCb^a1vSR9C(4JsoM3uJg3h}C?DY4$5 z=sNZY8>gk-oA}lHOUVNMfa;Q7IyyZza7HSs16U9^;50MJyJ#ZqMPlfOm@8Kk5Slk1 zpTZs2oqSmHwby9Xr;!OBnlzI((ZhAj4ALbnRhH>u=)Q!O5eQSG&Oupo@~FGyXE~bhj<8Co#i{K7AF4>HAgw!G77b+8 z;PSpxPdn1qBbY<;B+9`}6x+%Qsp+w77OwDKASd`8+to$_W^t1`qXRHVY7ZbLU`lc}QIYut*kOG_uYln*x~ zmYKQhlgmvg8L$;n*Vkx~_Z}|D;Q2vo&=y#+S@sBzYPwF}i}!m`86i*O%^c~}iI%aY zRE}^|*`u-(`xkZfr6m6YYm9^Aq++wZ^|xAnKkQR z_A~mKz9?=t>inLZoJ&Pp`Pk(=+fYef{;{w3jsGPT&v4Y0*W~ye4)qhBhnZ~ci@gru z3j)kE>42eIyrTeqmh z;rF4f2{6IAbr^;voT7N^v1GX$Rf83K4oiZI6y|w#M`8`7| zT18OmLUjz#)h)i{ZK%27y3?uYDW^{Qm_1k?rTl2TdR+yK9vPDpiPVkT-*M$2rP!cC z5a45)Lq4gX$-gP|x?o4zMi+I?e|1ssmZ);jQ&prCwY=oai@5kp6mf(8{1cm`kK~2Q zl*4{7i;OTuZ^_c7eS@+S0myhZIQAfxB3e=Xb?jut%Tz(O<$J;IXzA!I$F|2MaHk))ktv#DzXIvHrLZi2RH0m6WW0$lb>Pg}ycoNG(x^A!_>s!9(zyadkqVBtf zBCMv*&1x8QEc<0KyBxfEwBPCt7UBB%R;Jt>(JcmW%4FZA%a#qN$%esmFbUPGjaatG zz9`LDFTPKbMdL5U;;T4}Py!^gq1T+Kz*QtA6*7bqzRGs_e;ADP`o9?reihj;9vV6XQj-=QWM+Jjg&jNTv-<6&0|sk#+!g|^i7LxV z$`QrOANCo(o1!iK;my>?oD9$KeYJ&FAmNl)=uR&gE_kx_GP$>IwVgL1kMrARbQfBt zY4Aj-y|caxeb7fK31KAo`7|U=ULH(`MRWkA(%9`9o-UeF#?LH?cjWP9ZZk>FS?YVp zAGVaXja9ZnzimL&R()jI{(Bwzh=I#RljA@9*o|U~)8S+M zHN?8JXCI~q0CDoSzDVD7_4RS)dh%g4VO61d^A{gh*VWYl4gI%N`=WP7c6{V4eeFl0 zqrZK5J3Vt1udi8oqF?s2lQYg+SI1uZBeeb-+>e8c6Mz=pOOHj>d^@{WJ!`A$yce&uzv{JpraWc-yKj=aM zvMeh;efapdBMaP&PfkX4GDXie8?Kn-B&;BO+R5$PwtcRy?)gA;KtRFt{rUkFhKcBp z{~UC&lOULk3(y@rsjB)xcqBHBdoxuMsDpDWQseu>9z+9eI42aFHz%8U4<3UI0^UUZ zOd`(`O&y#H@X~|4!t|H^U;+3d#r4;q&skj7L+1s}HeN> zj8|%O$=unqcN{u20>32_m09S22bZt@gNXYG5(tas8{4a`w zpnjPKPxj)}EcXe+%Z80gaasJc^6r6yrF&g-cxbFryRnOlVw2%k=0j3FII{TJqIn244FGCaB}Zg}4)DWNIp+>*BibPEwd zUxX2-7V3L`uKN6N)$OGZ&rSLy`Y>PzwYT-4Ag`}~mZdRq+e0O|cm;nI07BU_o|XPh z;%OUMO&a!|4Y+l17*Z)AmSa1VrFl)!`!&ip)hp-Gkp&(FWqVyQ8tgND;vG#54ddx= z@~J=U(vTEMT^iFeG8i_b~sO zivFMxZ$SA|VX`p`O-b>9xfr`u?Xa!(O>Z4p6RV!8!~-mK@?Vwr&Whc@^DCoUfovY` zhvVB2IwIjpBaJW)c9PsPi>~7cAZrr1dGk06i#Mz*t|cjejGS zS(9kz>nYGKCMw#)=M`D(y4R^F+EsIK^0bp>tTM(s zMw`8Rul~AkR`$<7#%$$ozjNzF&Te*!tqJqbqYn8L^jR>q*GAKpZQD|=7zCk!JJ8!Y zctY$kLzaFo7_U=OTG|hcN?U&EefQz;6LjtXFCQ^{oH~W9$+ySrTfi5J6-wuyNLx4| zURaXk4tt;0RV11kGW46VuBS8P0PW`0?M?g^vhCRKdu=woRCvX) zLFeC%OMYodNi_uc5o$SZ>X zp|4_*9C>Ejw9+yJyRQ8DcOZb7%O{NcWxpZ0osDTs#=gYf59+m+8L+okQnrT!*QkpFI=Y34hu!U zv@m+dERR|3#XimPyKvzr?}ZMqA>4VdixWEd^#pb3eXqZ(G@uKq5$N>haXYGCh}@4e zaGcbT#2J1t_Gd0l%Adb}1wr*1yk7MBV*6VKxZ;W5#^Lpy4U{iNaW!AQ+^cU(YcouD zJ+oE4KjjT>-Q6NOwgeKYSM!Cte-3ttw5G6hJ1c8+V!m-C0@DH%*Wl}r2lYi$9N?Mg z4yH+V(flsE?c9~jMyGlO&VK+c?FE;1I`)Y|(>InO)DrV3Cd94U#BzehTTMwOq%I(C za-4YNXBPV|+*1=*?7}vt%go!^J{=)3^b8zl-;{J>w=l6hdzMkm4-)2%E%`;W>+CEq zdbC@)?%8%wYM}E`rBkO`<}q-jRAC3)+OVy}`tZKFGu)4_#2TJ~^t5)NIF&W2zzGJv zc^Jg5%d>sGr=eD*ErIWgEF;A{fKr>KjMoSD?0xafB|?@S1=UT~4k2h2ZNJnH+2)Be zr^7Y=-Eh+S&<;={ zw5HT3xZQiV8mq`Yp-PW;Np#X~6yt5x$7?wwqRa*25bs}n@%@n1t~6#aP4I}*i#jus zXg@+_ql9giJ1zRbN~@HBV-wWd348W>{_P>C^dTWoaB% zd@DSTO@s5_(j`mqfPMDl$qw91$$QxHTqM?66uPale=3SrahMgfn-FE?7d&)O^x@qT zJ0*+=**4mK;PNgwHIGTo+Z!; zcOG^sEnqLi)`jY_6oFdhp&zA(m*-r99Ry zj_jD{u$)aZF7iylCwV)MsPqSWwa3v?>gJY5KaZOfPJgHf?BYFnYULW?bY)_P?EWh!DrQxxEF?PyxAd z;1wh`t`KM6&0CMmiuJO(uJ${Nf%y&W75s&g;m5-46@-z_^Ia2pTwEr=aT;3mUbK<2 z>R)t!nR{^91t`opk{?G=m-y|+LS@3f0aDUNrD07@C0zzy&JcU5;L?8hxtNabk_S0?#GY3@F0^t^G_5KhJ?g;-@eMHPf>#g0}vN~VYn$Bq4!SB{7L-h3MgEJ^y`-o z41+@b%Al5gGJ=ux%e`1rFkX=QlyE2j_i;y#@JGR9xvR>f=F?f51jE73+^lYvHxz+5 z5HfDf%@Z{L`{kaPo^mTQb2hYmoBO*POF~Pndl5t_l7o=>!MIzn~Q~-puYR*_Ky~X!VtihQ$_qp$N zwg3*V=85uVZc;xQ`!FIZb!E|pBK(5E>yoCxNv8HXEnXaF)e}d?1N-;KemZXONwszB zMCz@lPwT#2JjkNpkZ+ynF@&I~n$_2|i{QJ0BdIvxVg8vhBwz!kG#@E`!Y&@l}wADT?9^4GP|2Ez7+}L{dgVsArh^;B!}wx#?Qy; z_7=CP+mgDAf}&zaT)rqMe=q9dr`&Ag>@$*)2gZg)4RAC&bIrHsk6m5?v+YbJ5{s)q z9u6mfsxYHYnA@njkFB;{14lt>(A&LFhHp%f^Mvy0MVsCo-BcP}XZL)&Xua)zgo6w6 z5kG_(Nek~#R^;wg@`#%#4F+)*fpcf3dD^{MQMU<(%Q9@>^#>0`H-|yk1m+*m33}~F zW5@0y+jcZCGBUz0)vzJKA(RBFD_h+ybp8wq#lo|ie$D#V4)tvC9rPxFva+_1cMujU z@Zknbbyi=>jQS765FkO}R9ipO!}+;7(as~kP8zw?tX|t?tY|vhC47t^DCaF$&;s`! zp3R&9nZjs*lt*R3JUrqcb5NbrBG!R&fZ_X#xIGL{82Pg!5GBsQm9S5cyXU0q=h$j) z-Qt(Of2)liXce2wNBD&he~4gh4Le1=1Xp$I+Eqw0ONZh4;IAR7_!lFTDsko9hd{h| zl0X&jxQTJL+NiGf;Sf?$agH$2 zclucfy_e6QGX~1z@&*MqL`CiJp+?CjSPPsXq56sKsc_!5 zulMv@aIC$4{I}O_g4zsr?J}rvQ%(Gwr1!?nMIWKxB9JdrF4$~h;?(qHXWpom`zXIV zpV93Q?dN@^uIr$(fwM%>KcB;i2n}59(XHFthv#kXJ|-+u31@TM+M;u5YSE(bL5uaF zv?y0o=>;r(E%%D&1fjgE#)IONfhJAv-2`roqQU2PWnQe^uwnJJuSW2=k|`S@k@D_f z`vM^EgC+Q(%H>TYH8d5lb0(vLLj?ePbS;SJ^lc@nm~$x@3Rw{IisJUgJ>vXC4d00g zFaXJMk=A#w-MCRjDUB8WhI*;pD=Mhh`DcgKPd0LK!o-D(^Vh)VipGo<&;|Fln&gxA z;9&fcu$=70C<2PV-!kR8RFp(m@7S>eMG-r2?Vj`|5v3N6xxRTbM!%EpsT++OH%22g zdBzO%NYya+Vox0{U;AkS*)){anV=V-$~0mw>I}R?-jyU~-@G|~?97Nh8%^&-Px`~q z(C}?%2N5%CwUT$z9fPS3$C@u?UlFKa?g zx3%{4eHMvJX0w?by7kerqwQr`G9j2XP;e#Q4O>+$v1%(Sqq>R& z5KrPXb9CCOsYPFRlGwQMm1Lx%6k1x4A=0X|CS(TG(tVLWz$>Sed&U{uFSyt{fz-)g zgNA;2|6rQ^Pvs#`8#wk*X~fg-9CXs`bPVni8uDNP%KK|rb#DH0HBOCtH!jR2Fbmag zXcG`VFO7~NpvBgXgWu7)bv7ZPrD4yEYVbbOJ3i0|Fm-C}qq({v zB;qMGYX&yIH(n=gAfSW?h*X8v^JGr>u%@IF2zJaYEx&yD&{Fm=4U#ZmUx3|UkD8(N zprs&n!{fgOU{_AKL>iUsT+~W7v!K8QR0?VlTkGu5wck!#$B?GPnI2y4ER%PTQZXg| z*g2G-^6L~NO=})CrsY*vj)f~8<+g2NE*bdw#0fPs`MY;`_v1d*k`X=$nKDbvQ}};N zGEh&n?$Tw%f?J)Erzl$5W<33G2+uAPdK@c0=krp3K`SI+OFRykqcv-jFRLz4`Y+&V8hc*9Tg zdYFPytN8vd?{#ZdOC-6JL~cPIvKkOxorB?-mDsg|q-u-D#an~v$0S!?efb;KZIb_dSEkJ=xzo)HdnE$P7%-q-{OGE(8tc;ImJ$1xq4 zQxQ47UD;9s1fSTu5{a`EAmvWtjKC8ng_8(zN)fMyvcgvNCayN{m?7id^pvz*Sie3$ z;qZb)=VfNK+Ai7a#rN!H$Oa$-E8owoVmQI|NpjIQ{x`amE$j&q(_YWvKiI|ci_IV4 zEH~cz%}0;wIWM>jVUlXUUJ~6C8l+b*;fR0O3T>_7(4iq-HzVGdt(bpFcH+FocXOIw z-aT2CI8)~2%T9|3L&1PQt{QKXc0sCz6gU$8XzVznRqml~tF>F3eNnpXv?=etX zbH>VMOkLgKL=Ts4nMv1b^%9K-mptO^*WC?Q4FaRcoqPYj2L{cIAX72yDjB1hneerh zQHX+;S5uR|=DxgtsnbAxHr|08JcO6w=msbhqBP0cE-%p-esue_`@SQ8Oxa;5h@i*m z5F@R*)~dL8Mmw=6e<6$GI(UtZkIv&su5nKGBx{FsN={6C(11{`ZkN}!QL4IkNI!n| z{B^VMD17emRe59(;W0#bwGucM&zr}s>x11mHFI~!E_l|l+ICDcbN?sT!+M=;mlL}o zwZBlk+b$!66>>-3jDy$PWRhzu5OF13HzibTfQO@4E>4hS=hW^H()0Q>fKJ6r?u(M%7=)Fa72`PrY zi>j*56WkZz>L05N*N~L;MyuBIreYFCZL~y!IbB z(8S~%)a+`LMz>85TP=Lq&Nlh*&*;PY$9&tI7iqDihVcT)gjezrXkye4o86iaa%*hH;$bP$MNaas0S?S0OhTjQji0{dm<X80C&@i^i(|^8T zv9fnCTzG@0;@0pce$I2$EhUob6zUP6H*8;H;0CFFEP8Y)t+-{QkaH|oz#*gqtB0ut zZ!E9ily8fOVp^F$l!U%8EiW#qks0A48sqSGo!;O|hinuJEztm-hO4 ztMN9RYr&GjRgtflmFD^5k-w>@P6yq`F0KEJ)|EpSq?r?$n4Qxu}HuUi+S+Az3=aI9Jf5fW|Devledr7Z*M*<+-}v# zep7bnNVV0sk8rslv|nIC$P|A428 z=mhp+%@O+!x-{}O==!psXN904XI!A1^EF&_Cpkode{WNTdS;iFl7I2liGrq8~>`XJYh{`N{M1O!Fpr zm)g2=p&V^zyqK)e#DoM%@2a|FPxPnd9zvTz z9?`#l0@xmma(G;1E5n|~=jc~1S?~=+gq>$|Ma4L~xIK}PFIW{6N`*K5(iQNvk=ExS z^YWJ0{zw8{!5g1DH#KqT;>AC2^p1b?@O&aaTN+eL-$%2J0Pr?r((E%H|CN0>)fb?I z&@d52)ZyNq?(S#NiE&$`V%^qB{vAL`a!945IAn zAGay+YJRxU#CgYpwbTxt=w(qcbN!%IZNBgPg+5cVNWjd+@oDa$X(}lE1Nq9VH-V1F z{|6olIm?df`b?i2w$#PqIhCbm08TM!(E;BL_c@iB`_>@AcxXt(@yF z>33bHtDHe}FCgML*%8jy2nTROLv~^A_Ok{W?i#mE{d=V~nmF3cxUI$9G2VH>Ff}5v zaqL5s6z($(^-K}x;WK7KT)EW0P&jkcI%aD9uLUosEcp8< z&o~E20Mvgc6{pplA*kN8&p=epk!zB;0)~$>Ka`q|Gq}$;(^av$07FZO2j65fD`(3ZamF?p*Qp>oH^kbL2jM zHCtie$AOyL*xpfXqc~7SaEy4ld(R%``+8DZqz{5B(R94=qN1XTO&S$+%iLY=2gKA+ zJf1Ymk(Bn56mJ>3W9Tt}{1B=Qq$Rr5VU~m-h##`PI(Pc7N$R?-Y;HV=QLgCzQ<(R- z9Ge{HL7>GU6%{PBKE-B>uz}`pA|%&$JeKj5Ik-7N=8W;5;{-CY_yU0FO3@WWNC8zu zmoa0m)z{vxXAbBL<{Z2^Cy~5Zw0F^o|BG1)JU;u%CV0%DJTr>8oX}u%EGLEEx%&(< zGW9VxxtI?zwz2Rhlp1uId6}2@mdZjHKHOLA8!1@IafXE2Ybe&*oEQOG^fZF7+>Ce` zZGtZ$NjroZ#4Pdv>IUuD#tvGOe%N&MY%@4k=GxqGubZ6FKD_Kqx>zQ21gM9TBCF6= z-HardsTZ+ualB=t1K*AI4=((#MY+qB{0fr3JJwFc#X#!0lqO-2#TT_xIR3PilDV#> z{%E(1)oQ9o#nnyj2L7{FRZ)qs)#j-DL~K6^QXvXcMVOaqDv4e*apoz9f6;GdyOWbY z0|iJh^^vZSWC-Qo(g*G=A`b950Zk=-F|(7DxbJnGY`(uM+|j1Pr`x%We_t4cp$~-(BVHw&QmFgp-PlcB zi?>f?4wCVkd>&NmlD&gjSOY0rC4u#L{>UGO7#J|=Z8_ybh#^(OexZsA!=v{)n)$Ch zm^{EeWA#6tm~JmNJ5>31P?4gdB6^TPz+m!68cVpT%3Y4y{W61PV`W_Cj+PW{k#xr`)ZwUa|R7Z*Jaiw*jYzM)1Y zgMhW;tO55jtm>rY0R!H9YpqPV&ti?;N}PO6S*?YhV$EOaLU+rPle-wWR^WljOzmI} z3@`x23KV{QD1n%%0CrzCitv(J(HyY;N2_x6{G1qOw zd!PsGeE-O(A6wP)CwvjF&q^Jr2`%w}A0vJ5E^i8)?`|pPV_XIM1*&&&8si^xPIhFg zt-B@=;X3~_!bJ~Hp7skS2}SKnFpl7#We>9psk5wo5i$j%{I2eiuH*~nnL5l?dUWkNhZhJAc1|vZP({W>s|DxulVO4R6Mld7rxPIwBGyd&+}JuOCAEvVP9q6KgMy2?qmcHQNblsXx$K217T z+~~;hh8O9-@=Pyj_+4XI^qUcoJjFxt5t$98Rmc$0uB+g06*sxBuWvmA21wBdFTQ8- zMlxySza9!n@?YnoWYZ*DTUW|vP_*pKp%fz%X|%Sn>0bT$^NYPp?sor2AUjy4=l*-m z`pNbRxj`%&J!%vVRf~MeT-@C|BjzH>C-b3*sqOBkYxJ*RZqaLbl)>*;X?%nC{^k9n zE)+45nOIqA$ooTzfiA`TAGBPE4GK@qoc=~_w%WhmiEf3IRS>L_Ul@J6F6i3eI%yLI z-@qFx_~Xd*8Kekf$?)O)2M@vr|8XPiA3@JE_IB}KkMGQ`kdV*RE!3$);=N3C3OfMr z!J*eeUY-X)+j%%}IJ*DS<^S1%wsKpucYkz`l}Wu`N2HcGy+tmFZi0ATja~#KKhTgd zgrYb?3}?BK=7x-0*T+n{9e=;YEC1LNS|*X!R5L`>8sh$DDwf9c=O3T%3-<`7EhiL{ z^e`AF8Nu1pYiH(~o{&i7GlUl4&)A0F`CnVyaVIpJnDEQC?8J2uZ%cn69f!TAaL&K(TLi`7I484dfmkbx_@kih$jh397VZXpb{B0#i12@P8QUva| zZ2x@Ae}#;HL@13OB+KMX9Kym=o7xrQ-I4;}2w2@zxlepgq)Jvh&0$x=8eLcA_b{}; z#{1e|ph|ZENVjxcCgmuear7{!<;&|)+qu1aXm6ua0bLZ=u|r713s&n9e#F4h==kmw zo4=x+qnW9vWu`8ZmP z4F3CQ(6uiqDPcg$N?+jgIGU)xL~q+RK@j@xZz^DEDl1EP`9zX?XkV`SZPTP2M_$jA zo#>R5lu4gDHSC251C7DQYitiC{Iz-_qR4a+B=P}H@c)2eLge0GOPP0ZoN@`*xDeF+ zI(-PRT`kdsKLW(M!F+6TQBFWbdHHA{X=Z%azb}2O?N`kytt@)~XTM2qzo8^2R8y{% z)`&6y9;^`D6di!~XSc!m&>2c2tb5=(CbdJ1!eC@I``P~Dr(1@obM7rc$j5f|#VMPw zTuhblaKGOWhm~WY(SKJT=_@}Fe5fm1qV_(JnZ`j+RxR8v4QYWB?-Q#=MOd+>N(opsA|oqj8bhj`e-_t_U0LIfO<#GHXOQ68| zi)(fIk2H@7gI zH6Oz3C}+01I&OONyV&_~eV3I(ph*ao5IvPNGIB@oSHW38@q}zS{XFZ%$cFS3^U? zkrWxoH%O(UBrs6O9wfP_3n;~7M1YN{7ODlGN#*qcrZY+Dl=WQD(Nb6 z-f#$8MfPq9@9BbO1`d<%4qjFK_hN~$s~~YDjz1uxXRZB^g&OzmL7_+c`BPj5e;kok z%l`V3;KtH!P2H8OVYVNS(x#`Xs0i80uJ!gt)OJ<5x#XDtU|)V)#%@uh+BUen0t9xv z=VK0=dsF-%~Yi0KqQ-HM<1JVST|SCXyk$H$58`v)=1RaN@wWt_&ubL z_y6C2v&jC8)1Sv~%`LWR6x@B|w3BTO1;jD_!fB!DE`NrDj=xB5NI85S==r%~{w<%E i4UhEy@o(PwHo9ay@VTbt?vdh$%|}~~I%VqU|9=3@pBa?^ literal 0 HcmV?d00001 diff --git a/doc/images/fcn32_16_8.png b/doc/images/fcn32_16_8.png new file mode 100644 index 0000000000000000000000000000000000000000..bbc92b32ff98b2ffc82a66468ce01148cc12c20c GIT binary patch literal 109595 zcmdRW^;?u%*zOQYhk$e`2uPQ7cb7`1G)Q-c0s;b#lynJ*bazWRlr+*U-3`N8v%hoB zFXt~fY_4nXLHE4xto5w>x#M|>)KHVh!6e6oKp;4Z3Nl&{2x=(=g0zp041V%Pm|P9~ z3)NauRT2WJjKjJ!djww7nk#6jLLj~j5J>Px2;>I*DEJ=);=u`l?3h9zLf;?|66f?L zO%d=1kKZZF%Rn9w|75oo#)F@rxhty5qOG7(BXJ;cz7A&u|1(5UM)HmK-@OHEQ_9Ik z&Sf4CYd?;^&uu50EtQij4P47-dPyrf@qHWVpJ8AWC9fM3&|>h1;Y*=!DBB=Osc=!5 zqtv+^swTbAtJ?WlR?|8(>@QBt7oGZ6Ti9UuR9Sy$I5T5%HpOdL@G!c96^j-<^nbpy z6GB`UAphqTEmMIhCAuU&>VMw}`v32XgdKmerH&5BbZIbngRhjZD(bk1UF}~a<0miJFnwm54b$+f$Pf)90s$G0~azf0e zZ^4)L1Qo;SVfh0d0qSuF3f|{&QzF?{*y<2RklGgg-g05B~ ztcTOLcX!uLueZ0iqlwv?PJXjILhk(K)x|lu(jBqnci0+D!f|nS*1vHwI5_yWSewnL zDS>{#=GD_Et{Eixrwh2Go34%Uo9T9P>~IRlhG)#edpkRStF1>47Me#gMd!xG2+&d6 zPBRYg?zY6^d|}UUJJp1KGI@?+k+A!p3^2;aQSa~Wo_dQYW#fi6hAHVZLeAV}@{juRCkLfk#oS2w+qpeMV`ed=8tqoq*eBgV%k<7!*-QeiW zeRO;8cd_Slx!|=FaJf)i-ga}@{vbNdwo2N~*%wRYce`J|Av66ZKiQKO_2EG4=Xpj+jSN5Of%>TW` z#KIE4T1F`o5BjIfm$pCM4)?IM+_-v3K@Kt9_?_5(Gg&Tny&g|u?46UH4S_hX{=z6r z%bK~lxiR)X?iotskM3DL%4vIm-!?juShpA-_R$30?K=g{nR0%_>!^>Q_3aCF4s-DPTRcjEql-Bwum>Mjn$5dq?x&1@Hf|TBZ4RX|Cd26B?3nvP zf^O%Qe*XMPgekj@6ymb%?l4>R;v`rpUn{ksFDPo#F5sYXOG!ydO5F#3y(Pc!9+WP0 z+RU!@riaAj?Djs-@qPVV&^62iGpwt>m(`%|{frf_>XQ4oYImFcpw($dN5`+F6L5mN zNA4F614(*5T=!Bqee`TL>ke;!xIYBNEb4uH+i_39u=#9Kg+$7QrzAyPU#H}7}g(OBbzFAqb6Sasgk1yNGPq>yo51@VcC*Ax3d6c&hg}zH?b(vBGe=Ejc#U?q_4_bFZ3BJm|9#b>=A3JnSX74UpyDJ(3^e1B`ZFbmhg zZMU?ZkC*r3l8&|YR@TGa7L+goet#Gwzsn|3i6(4HgtrNHoS0SD863aXtJP>c@w=n5 zwT{b0KXvBB@QW`=NuEc`olJGaN<h`blT!ZJ z+Of<=evaa7y_3m$>KLf&?g%`S3%yR$Bd9!e&U63o(^)4H4r7tyEyT%&1T$v&yUwh2C%t!^sp#A~RdvoHj(*|K(Oy#W ze!@Y!+!#mhYFj->=r}3DYS@N1m5F))Af*roM|o>&KnV`~B)MIN?`pLtny!BP@Gu#+ zfgaidK2+YpFS}aY=G~;UG^_|zLXy2;&58MWN}rQY>2rKx0!KxKnV2R8))p3@$7^%e z9}C;QxjTvwPgkH71?%m2H0RV#RjL!g@b6b7VUDvXJqL%=%-O=R=-H_B%2wOOUwgy$ zH2UMEhx;YX&nRFeyT#$hafa=MwzWqH3={bkqNv6H-1U0fB5as2uwDG56Hn}FxgEaS z-qlV_Oxy;)*(FqWEE?Z7_S=6&B#Q)LPqtr;xdzBZk(ne2)j@(7dTPRd)z;MP%~Y=4 zUoPQsc+zaAE!|nIn{a@_bWPJS3Gi@sMqKT@$zxY;A<~dN2-LvI$w^lnFCSl~kO#BA z6Dp$f<@LYki_oXm($?nNFcQ7pt6sX>ub=k-!HZZo>kB7Z{Fa%h-9)F z-QU@1+DJ0+7o(XzwAE+(9G}poNPwocWs^FzxdBU#mdQP$^K8CPGH}EIttvwt+(`UT zQ=Rwm>gZ71al1_0?ZRPr;rp<8R~Q?ih=a^M_T?`^b&->PIzd6fwf7%#|D_AMb+yY; z*H?g(?shQWD1JSl-a8wo%L+%V+jV~k3R>>DTsG+kD(6iV!KJmrtU%up(aVoP0ifr0 zyRByvp@My`sjCYQw0ZxYG6;6${pN~EveE^{Rxh)d69_m)pOU^9F{)&y&n8)<46-RD zY0zEZ0{}xmKfgvt1>|~8t*^gA#%tWRK!<4Ix!XMqyljTU;Ya1uU?aQ2unH^YoB&>u zhQ~a-_?nh>yHi{)mziV4K3$-i79JkX-uy6nzpy>OLA80%eAou+l%rYis*OC9c6N5Q z8~%4`sU5(b$=xbRa_inqg_-pFY-@2b3)n~{Rsd@mJ?{X2*(D3dxcBYG;Wu>Yd1}ay zsbpFaAO6wpN2XYSZ@N@ir)foLDchOb`XGr-!Tw^au!t2FL_t~k7SVIU?)xpOMt%8o zk3|*{1wY>&BtBiLutzKx>ay$%Xbd>!=IeCO%Lzu=!tqk0=8G!+_$fqlGjcUIUsL5@ zAZiPzP@d>Zz{13Q&iFm%1!aX%!1<(7zUz#MI5^K=zkWU6oi_0l7l@UgI>lJ<`=DjW z?A5_^^CvEls7)cC$!_LZSoXHUh8j8h^~SmH!&VU7x@5@naFGff6#^j*fBW{WY(37X zu5J?5Mb5lS9~E>%g<+%Z_5ct+XA#b_yVI-(|J6vw21nU<_0HcCVo?Bux9hVenc=nF zoxq?AVmv%<17Baf4vaK#YJG)|{&5&>4S^uO`?b?@Vke7!hxa==?W;HFF`)(9BjOK~ z5L)7Bd54v$QUlNpIXkPJEZ$}vCmVb;ZUT&LG)FpzT*HtC!=}j;E&bW;<>AAY_yZaI zoMP!p&O-m;E0b<@s#qCldnCy&X*BPP&6AJAHP8{EnyYte->;iv zxGi}ZB*A6BqF`KJ-_{lg-VNM8fOAc!t!d}PY=)*-N`rLC=05a;EN5VJTu z;j9JEIfrrEZ%c>bE~~$O_>@VfjAnYTMvj)_MwnZ~?(NUj$5o#qPy`aZTh`2;6d`n$ ziz>MXn%|9Iw8-DJ3G~HE^FHtrBlHVwBS)`@InfT5#HKEC{OfP6{q60oxesbK6s)J< zw4s}7gDBx3Nt4T3Z@-tz&P0)I9g=4UJ>-9%)WUFNh{9L@hx^M2hf@SSKAQ$5cI=0? zr}yV+$|oAgm#i#txAGp&!Q-{Q!{$S=7&*cN)uEvwD9=&QMRiup%`dM-pDl0>sgsP6 z`i=n)5IdiEoyV{c02ne zmj}sC8PTJ!0_aNV52EYlbZWca{Q0){xjeGW^FJT(#QvRLQ+qFm9DRr%A|Xdd4Lpg@ z!cMCw7~g)rv{`|APIwob>tR|r+BI7LT0y|X1~7@%%SDPtE6nbS;<6??Eet2d7anUJ z8eI8i)(wC=_LqA@I+oxU^<;LZ6+A$0fgMkPZ9vP}p$a>@yM2G9_trW920`gK1$iD){%mfB(MoM9tRh#1(L3 zpg;p1UOw}~kd@oNV?stXgyvala9KOqfblym-T~_H06tm6drO`fr1-~s-~SMl(b3>O z$^vz8biIpc6)#kc8k|=I9z8Z2NyXUM*nrXumDVo`5tffFF2eui#SxQ`;O(l$#Kf%F zcAFU)89_%%%gf7)ijW`xgCb}Guhs|R!H>~!DX$RVfQvO`bqY9}$ce|kr=Nu+76ddI z5&XuGrNef>YJt4dF*2fRoULHr^K3Q&m}6ho$WT=y$An8sPTq9TwA*0=SSTfKpFl*< zyUO=76-W?TeDpw215|>p-}Ca;5DE<-t|k#1JA14lJMXN(Q9qOpmds(&=I^7gukXzW zKo~}cP7TGW|IuesK0Ttr^exxLlotQTj~}G7q5w-;%_0Jy zhshUOt&_!C&Qb|`VmD`K=<3?k&&e-He_rdjzrCc0kqePgXjKwS?BLnJ%65+7|^j9-janQqJRgMfaHUBIw8tdyTX-@FB z9mgOf*h&PAag)34x*yrDP>khlSy`Ee7=5d;iH;8cUO51vsY#vVui^z-{FKTLfsp=lSfsJi3Ip{tHbW-A(L}<-hjjI5CHPZa8E%q%%m? ze=A^TqJ9?%2?>B>83)~;ivyqsWPEXS;op~og9F5>Y^FI;dtjjL1s7p>j=^raY2eO4 z;>(MjfaL<7K-T@9HLx#)Ja#7X;>5@WXSlKV^mxqHUCUu*gXy7Y)ZQH{&e*uPa^X0v zu*JaJeYefQN?ld}X-oG9Uea+CF}=>UnE>=bpT)z)MTP)g&*y3iMC=22Kg<_{=A@g# zOV`5)?PNAbi>{@mgr;D{xXZ=f ztUZuC2vY@|RB+A$fd*Ivt_J9Zj_U;f00LW2d|ep-s2)V(@(;KTj~+ehrnbc%W_z74@_$GflMh4%z!FQD+pbkKbbhpq%i|acD zn`d@=u}|ZFBHrgn6R~)9R3h zlkx540oLuk5(aD)88L4_BMlrIU-&0J0RKWcf2{p(gQNT6v9Kw@hYWWiPHdok;RRYn zf%m&65j5BfDg*uf{l9;cKWtZ64}FAg{FU3}p^*;=xVNlVR941fn_ed)T)epY@p$TRZs z;P&$TJVU_cokiF;gaFL(&8!Dp8gya#fKt$=50b%WyFT4wgh#5xNYH-kLL*v05q=bb zOLeos5hTbu#}-P9VAPquJ4Mb_Wed$-)>c+>m-+ELxM$uXB3n##M1c(rz%IFXZ2O8GaPc!BuWSJ518 z-tfEY`bA&+^ zwBBh+tjhJ922)9C=|Wq8-@oakq$D#JQ)%gL^EgrB8=t!yuG6iNI7v!Uz9KxERjVJy z6F?Zql}by0UGI&d=zc)SnK$~An0_j?HJsrW__3*}=>r;G;87SKqjb`SsSmlykUfDW~d#w1m=wY43^6f2tP zUC4{yJb?s{Y(2C;Ts}M(w*G!ZyAnyr7_N$nDXDn|+8LEk3YH6&r$iZc8n5f=5P#gH zFrYZ%57#68Wh9N-&rkt1dn~+tCYUiix-e)~SV1Qr3;a!LQ0XO%U74>4pF4$w9eKAz zir;sHpzQzmMMs#=YS>VbpT7ZA`U)*R1poJAB7jQVa9Zc}s{Q^q>2yy}UBJa6g6;w8 zP^T&h5k4mr%V84IzW&!QOistR1iu~$!j&DitMx7E?D*1;%FM-Cg!jp9ZN8c_MP((+ zj8Z@oIV9WkV^>#~&*>&Ywg8fTy**wiFZz$p`pKa8N-UR!l@&sEL#>km!%|J-ivfpG zkSPii6**rk1qyZ9NYp`e8C6wPaDdI%`2e(Zg0*32-U58L9XO2Wp$P3kX&;ITq2<1u z9s$&%)S%we%L{LAvrioE!jytUByNhf9<|^Ycjs}p!F7Fo4SYT6t7)TVPv8{GIzs*j zR#H>b(zZsj9)Qg?@3WNwt_X!WB0&}w762K+lZ~Y71Pdt#Z&u4W%IGxp2522&`Rjg- z=7IF*4p-7KV>Y-ZKuM9aYNdQ~ZT5HH!KVoe!bL?!KzHXK3i)*v&9oun>p}_Iqt$4R z@Ibz_As$|BzJU1DXOvFQlT~9%HiZHP$k2?F^h;w)EC~2qk(hL>1t1+&mM}z{lo4~? z983WUH@-XJsjaOoN9&oE1s^@61GE~3T0(F%mFh0QR*9Y5I)2~T+3An8LpWZ6f$%K& zSs_fDGJKU1IK>t-{VEOYFx@)G1@i0N#a2JMpfD=VM4*;p0WH$e*3Ja2)L`jm3V6=( zDJj>p)>)#U0YRrbLD#vzIkz6kq`G#WskZ9s7jU+FNYN^CpXr~J;r{e_4KM<%2a)%c zOFxv9mFeQKfzI`N96Lq-nQ#GcFxHxXE}>31siFM_M%djl0S^O>V$RRQOFv?e2@>|c ztHdr=AXNUZn>MC(#kBxCCowVcaw+KHO5I(65;U2!tGOSOJDSz*v3QbSfQ@hiP!tW1 z8e8-Pc$zAA+n_PKjfgrpI3S1%qS2aM)Mf%$l8q(~n+}@WzX0s{c0}xU&jk2WW@Irr zzmSEOCxn)-o=^J)moN2EO;*?`Fsf&MraX$E;_Lw{V#g@zbF%(a$Iz^L)@mg4e#Sg5 ztn8}`B_jB+x3fbmA$2%8Ik|qV-DC=nbvTZE2ck%<($Y?rBLQxycK|G#iTXBR=iq?z zAOhX>#_o=ait384r>j2eN$WR=fR2;opnh#~F;!F?0MH#KWw!pIyJ`msf=ek-;kmD0 zMoxy=1UB>esTYDnBK8K9L$6QnkQEkN2Ss0rUUh*dS(Moff|WCqVm?o5QU8T7JAl*e zvpZFaPw*j-*d?TFfB?rghOr?4E)C*(?f)$R9 zje&B?>RyAWl7WL@HdDK1Nuu-&oxq3CYoZNX*z|X3hWmF!1?EV$0MP13wlo9cL)h!! z^WZ)lV3*GXMpT8hEJ<7+g=hS`lJk`0rwbmAQ1f6w1_)DXD1SaDCAjr(AtWKoQ@hi` zk76kjrWRmEmDDcLJ?_Hd_(&t@eUv@DyTRuGnD$?AY=NxnenAOVxvcGG2QbIDbD|uL zE0c$FleNQ(D*K3DaSd5n*&GAOyd*fqHlSz-7V|ri1)+YQpmG%@zb>BSXums&;a|K0 zZksT-(OV~{!$ck9Yao?P+=gC_i~iQ245o5U<>uy2P_p#ab$U)11u~w7zz*G)FjKUT zU0RLtz?DCGGxn54`}0$_f9_3SjeyO|N{WpP0lE#?cff<;cU7LL&b6QX`4+18HkMlS z@7x^hy5G(M#gJXcz+mo5&$?sU%}DKtAeu1e;phvrF*f@nWUXTbgbdkA z(Bmz$2n4xl1ar9w_JTDOetVA9)zuxmv8rsk9SeH!`o((ds>b+-jd8)hvp-iO&d~zLXmZL*~>T0!Kk!NPUf7pLeuEP@lI+@Cjr+H6Tk!PftfIf!1!R zdWO&>aKW$9Wi=5CLqS17M;HU_CO}QoyQPt2pX>#X*MLR_ zI42(4j}NRTvGhxHC0b5g${s+*Nv{!$g-CH;9+uYDz7l!;LhE1MJAu@u5{>XmO%i(N z=l)q9cYpB!O=Hn6(I}n-f?0Dt;Zn14;Q+Bc{!>9fP$6u;*m;%0@z&#B3WWXM9VcUj z9!x+5K7$KzM?Xef0RXPnT=M(wO!6w9aO874CNe~&m*e#^)ZRg2khxw?mww$z4Uk%R|Xky=%~ z{Z(Mg5XEFT^&OX$kuAN1J`yD&fOEWrs>XHW_o4rmFf?u+bcvNSllUsYdx0~L5Epb0 zL`oE|ii8S@WLvK{)9A)M`;3Ev9&)53AP|s^4FDLjuLg`QkwxdkyG32>+M*<1-i#Fn zUq%xLW*Rh_UF_yxo+iH>{yM13f#G4?dm-+TXGF>sD7|>C#_hmI8y(?GW2J4G%wvd-j#Qf zULmM7ys@(K7*M&j_x?cT9c{s@v+m!|a8LBtaGA3W01E+XKFy<}Rt-|rlagZu4GJ|I zODM#Xu}c1QU=vh>yEjTYXymJ4EQBJ2&&bZn9X%uXPb{ zDXEJ6S-{AEExP0}Z3K(z1yb~YBA;eJDe*#n5Gv^b53H>W?I<(n0~S%0x0(9pW!n0`}(DdIu! zadmMjI7pQ}*Lq{(F&{A!2*^6R&6;HaJ_ypC@E`XURTo~h>vB35{MaQ1;Kr;rp-*`2}yjS zt?!)D6ayq^ebGEu**i9|L!+-Q1zKY7eA+>ezmN0h))nNZj;_ zGbo!)fS(!toD(2R?)-oTG8GGtDcA)A<$vaF*mOc0fC%W*89E$X+_tkZWe6lh;t2FD zJQfWM2-C}#7!UI^Gav%Tle@r88@d3F%@L#cwW6M$-c@I1I1Sy0N|z?x_3X3$)EaQr zAdB_WQE5EU<6nf`7vKgFV!NN48Wddmgg}t4MPNakI9%WaB>=?XVDdqz7LO_FzingP z;~kD^?2p^7dKFR_s)|V$Jp-2C>!$hq_8ts)j8ur0>sGFUmASaRr>n<9MFwx#0!$Y! z>~5Dka6^%E`_vvI&}bGwK={Mz!U&B`ycRtE2z`cf=>iW9KqIihD;z-6{5if&PcQ)X ztoLu+^sZ5>?>P~i!o+twzyDqYM3^~!;s8US z^aYKp@~a+2QpXh*6fm5zW~fci%m}!q0X@x9BPlBzHoi^Shy?Zv)pN||8U#xuf7!y# z!0l3cq*-SL@YSgL_%wE}kY4D_RKHsTY_q4Q2l|jwZ8hWzkUKzZNmuZTx&gx81(3f?P9nm2Bfw<@V)2?Mnc@eSSy&hYwXn#NP;aLU zoE+@zguuKGA|Y!vrePit02`~1*w}dx)OlBfF_}sszx5&7M5U+bs0<#z*jXyqBqG4p zT^)n1dHUIumpm)rVisI!o2Ci{4fkVV=*P&ORhF`bRfJ#10cIosv^})m9yIg6pTRfA zGJgFUcz-F-W^z)}8Q?7d)14->5&{{us|C4|A#k03SPxZR`L}8G)-h$S$XUTl%E|(P zjvdYro?cvRv!5ThZNBW>^$JbQC*i|gnF;V@H5P)6^@fNeTP7p5`XI$`@Fc* zLq!fjoMLx3UqIPBn*%09J8-|Dnli@tGU7b`u=D`Gl*XY`Jjscm2I>X#!NbOe$*}od zXXqnna)4Qdbq}`cI3nrtZneA;KLu`U0BCE785@LWI9KO@7#snV@OiU?-%R6Xg!vhEiYUEjm(IJ;3i%Qc`jXnTVo*-srNbu4JXae(bgX_m$Uy zAw&Y`(?5CgeX|3gh09Ty;qJQuw$BQa3ul1@1IBIwazX9lS1s{ihx+$kL4xNTo7qmp z1t)$Yh&mKlsV6|Of2Mrq4QjY=;BlPU46vlcOM?4ad`_u@?v8=~I}lg^cqG|10nn=8 zZpK0K19F71E>q4c5JBOlH6!}x;O@EHL4?^igm9X%LKtQsASSx$(gl-B>82Yr*xl@g z4V**M??|mc!PkQ{?&^`^_}CZ%aRFY$kxz+(2y2c1Ej@wi&$l2#@og20vTgv*lKk)& z>RBA%L6^NQREuekM`TEZ*M@RUz=#ds-vOKOdt_v(#mCLXg)4SVDyO2LU=vifQRTZZ z1^Qej#9ttva>)J`LdCg5#kmf`Pauq;1CJx13iRLzV-d5Gl9D#;x%h!1>uMge;Z;s% zHv;{}rq$8PYEzk`m3KH8WiL1q*K>?)=Q}G5xT|2$if@AXD}8Zsae2dP)ZN{k##L%! z;yK9nM|nke+<m@70211#fuO*zeABavczC-kvmZ&TtIawUc z5On+2_q=<_#6)^-^o3r}&~X0GpU$d1#CaRAsM1_2&@T-H^EnZ(ifU`mfsaip*xB6D zatDYHD=d7qBQ!J=Q60dkc@5`fBUWZI#WeuKJ-|)PJ3&(2>QJi)3%cs2dG0nQvwG|e zOkFI83qS$9ONP)x(Nzs%Fh6EgibyF&ntcNr7K~>6Y;K+dHUth%(WG6gu@fh$tE(&M zjnAFTdOJXJ3bx-FFjyJ;2w)L|G6Jxn=jC;|yqvAes$D$E5MKoH;U7PkdJoH*n+@m* zB4wUKBxvz%92{D~I7Lr1i3=qM_UD)N!Mpl(L zNkxEv4TO?!yOd^ve(97AuT{0rVv7$ny4i{dRt4f}K<;qz3AjkHU%Yq$ZM0yIHv0zB z3Q&->_4T{wH-Ih3mW|IfdzFK+MLv9T6aX5R2Mge~)Cz&f9L^LK7AZYj?)*Tn2>#pe z?sQlNK%8_0F7URcSc$=r73MaZJd@q%7`DO6!|DzB--D~Dl0`2RPr{`B?@L0|HHrT- z_Qc8wg8 z2c|1-Q2LS!(j(~8|Me5 zU+_cP!F8tW=cO-lHT(`j1hh56B!S*jG-^U8SC*2Ve#SOaTCX zqvHjNP0epqn*ob|#wTV{7oRQOUx0>xYq^#YM0i zFr>Y*U;Xn}=8A$MWDenPy9n$N#&U|5mO8aQ!1LS0h>btHZXv967ToWxi$Aj9hQ@~> zdGaQU)qVSyZue7y!8tz^i&_ern$j!_ohc@^&o8KjFD4kXu>B~JB$)A8!G!0LH*o&h zn`z>foE+ZIANP>p5M%B6Q6k0!8F9~=&~C|hDbJH1BpA`3akgo`9GH%0=87^84C+|< zcXM~Du9hy)cMko;5ZVzP?WW4mHj=#JeAN}l6Jk0_Vrttv$Fsg${o%MO>2NlT zzi~Q^UPS_Pkle+Q{MiZ})6V%{hq_qMY9=O)usfP_E8ptt`(6v+hh~3&wtSqFn1J1A zI2T7r<~p(`+J5!Mf|vN;`D@Zoq)RRH^Djk;dFiB?@o6C)EPo~>31swqePh+Uno47K z>wXbM(PCE4lxR;A5wWtcXZ|%LeHOC3CC)>nJ9n->N=c5_ksDsyw^frZ3s zB4)y#41wr|=VUBq_o)lFEE@Zbu3^p0*DQ3u#rojb-kgtz^odDFUICRm#iA!F3ze~QjHkK0TY5a}7AD8mY-yG`(Xla_0?Lx8t$EVPaK- z(fP9J-~vc2_%J6t_OEu)Wd?AqMQTm2& z5^}WFJ)OXIrNxh%S9d<^Oz66@ak|7;r;~gcn(Mu>5eBJXg$2T|pG9={cQ_R=b zHcWpm-H9Ih~EOZO_U@}{;hmTmXDC_F7|SLPOV-NX_d{rGGdAvPI>)CjmwAPh(@(#3vLk)IYy1-U6gzpKQD zEYr9*2|u=xr&Q-N3n_%zi7#ze3+W{#D#%0=54D~SvsGJ@U0E{~_9 z<74#1Pe)fNSGdU1q$V8;e-CZ`o!|)}bz_Ew%0~ZmZ1Z_7D~lo9$xdc}#EcrM1^Zc5 z^@H_qNI`e@8*K~APS|LI6x702?lE|920ZSg{e?t#`F=_q2LO!Sql>g8pzHUJ#M^7*@rM>9f z4LqG_QzlY##EwmTEuHK-`rxJ&lKq^8*f5QvO)G0<)82wN zQvG3f?l0JLb%pvLla;0Mu=%!H!S80Rsxi7bK`-##hXZAW}oi@Ph?A3 zS%jy?x;&(5X_t>TzYS$&30teDyrX`JMW`uKOPZ&k^+Je}RNb8z!g4kT3wIxhb`Y(lPY(3iPTmEw~i}1E(s) zdu&8Bo)J8yh!INpyer*F7O$oEp4XKW&5r&nLA+uLxgKV=%aNk1SzsIXX7PIJr(8QM z%&J;_0s3!3&e-Dx9X_L*oyw-{q33?G{WX6^K@AM zsMBK0MW$cfV!)+GSXj4?V;>W40y=7RR8(2X4~2NxYqP{VPSU8TEB`05F1k&sFNr~0 zAur;66ems;DgPC;9svqw>U9e2S$#H@xp*y3)SR!P7tX$|w;YDcO;3{<2{ES=lB1VJ zt}RH;Q0E=)!QO?^=BQMMK=z#*#O}{Jsz>0v^No+oCC`d3 z^=a4j$y#m=cP28frVNcNlpvCcgX6T9q8vypSv55){Zu(x(!hO^yj?tO*;=jbpL*3A zL<@muKP;WU?5+{MCbK)A-+PZyXuHZqG^DC*pl`T$1e%z3(ylqO%OhEQYRT_XuY8cZ zzavYEsP{6s_V;7vk0zgtpdu?V#2|DE$7`OaKaHsi(6eo}g_r>6~Uy$d?|CI`NUDP+a?M zV$tO9k`nh1lF?=lDOcOu6*rn_8vgrwm6>thiK3GelOB_8cE>Xcmui>c>HpS@>Z#`8 zi35Y6`mEqqRzK^rmi-=RoLN{nbaq}yBBNhF9((%}q=eN?E%Q6>fYM)H5gtaNrm|AX z^6;kq+}pd1rP*p?;|Wv$+QUFNUHT2bQKO-th0^Lk@sVWmh&|bNUIMA-8cY_6iTU!e z>13p&XSam5kf>a5GP1|DH!yo2r^_$xBf}nf1;}gh$klQQ#dN=rAsx%v`&Y-+TO%fC zzAJ2hikMgjnc0Vn%h3&L*N--IkIXBFLBLIoT?- z{bM$(jBN5{#_fT_>HG3=e3xg=3&fu&{HfI$KC+zLu@dP9mJvYPn{ zZxZ9(no3Jc+p~5z2hlOI+N)|3cC}T|<7l~Gk(U`D^?WLy-rV)VCje8W93%ewwf5<1 zmw)oC$r#MIOFs2R6SX(Ns5+8=Z1QQ5iP5 z{q+?ty^Dw{`&QFad|t}^lSbjk3XNY9qQ7goIPA~IGmd15_$(f(e-rpZiykIL-86N1 zF6@7go21y17y`)9G)yrCzf&Wwui1XOEeL*^(bU9{#qrEsUkz}~M_8NS$ws258w&aF z8V#8VFus&)iF^Bn`v+WP^GgV~iezA$ekybe_Ke{U`j*Cm01@t4KqMd?w|#M!Gw)U2 zl{sgiQ^!VM-CdFSN#e`AB-Yv5tUJ4y@^Gi$cZuk=oU#F}gRfp#FF9WmwKBIvg_?iu zC^;;L>-QqI3$84FzCz9ShK8c%=2zQNl0^~~fy0!%F7T`ksg)Z+x3Au*>}NFCyZhud zgbm*%)PKvqt8||HncCC@=IRA{Jzxc0;{UL*8zqo?LmIxwGs!T;nd(jLUr-8rf|x5@ zf;NSaSmseEEonHrk-n&xV~W;G7S=(&l(8#+%sws%1XhLJjr!OqlfAgm{!3k>UTSAg z!e|MG?;Tu{go5-154v@yBGZR|J0~aeH8pudL&9nqRgDXj)bnJyM3Kx@)z!87WfUi` zL@2WoQ^?RCFA2GRFKf+EODB7xQDc)o##(wIVQya48zK_QG}4c+MKq0*f{&3E)RQh) zzPBL!FVr$Y3psMl$JQ2fjDb}NBFP#pTMtJ^WJ`MD(`z?3S5ry>LTBvwqw7V94k~^| z&`gicXYYhwK6^q-s;~P^)Ly%@(%f1!)7y6CSEVGw6Z^-=sFFScN~`Em6|BwXeI$Mt zlp&JKSI(3q(T|=!eSt+XTvW!cu0Fjc5WD8hgoC|Sm?FXX@4R^qttF{5cwxxfwxfzE z@>Ep`Rlc?K-uwmOs_Pd_SI&HR+ia`*TaF zs8s}$Jh8ZM3sCgNAkX4_7N_(Y|0@S!3NNprozvlFljgB<;cGzgGt`V;zZP^_EE!ZW zx4`|D&4GV*B}#?TCgwLPj}~tD`BK~`&Y4S=`In@6hm*%=AyV{)7?f^af`Rwd*EVi$ zUnif^bE>dGe;MGDGUWd7Zx!QUE1kAd{-m$2;ciP$kfc1HOrcnCOc}3 zjHzz8e!$Ptp)}U(5LEg`@NhB6{(C&2^q-_r!^w!E(raq|h9ZBEf);605JhF?de;S}$;C7T|%x%5gqnG&C->p+GlW4g4Tu6Nvi#Q3@eYjM3UP-PjiB9y#i)H=lz3JAbeFPsF3)pWlm;{}0a$|*Gm93&b4U7@nBZuo?P z1R+uc9!%Kd?(1Q-vbnGNs3OLCzCy4n-fl`K$IT`|4VAZ+)>%#NMP=3TYGr>+6LnQq zy>m-6LbZ}UjhKkqPd(h8jnjXN4axoSEtlX04zGNn*V8laJrTTY_JO zo4pH>z_%vkwMpx9$-(O63Q-(bgXZU7*VZNyD|4`#p}8m*B^Y^3{w3Fi72?ts^QQda(S%m!+-1& z0|K=9V%vGM*9$Zhz-;is=8z6)j1XlXX{t-@nYYm6yANgAeIwkg-xR`iQCD8RfJ+um zb9>M3uf4#JUYoibZb%(U>mT|p5gGyse%?Qzob1o5vYbYsK-mtQ z_s(~=HM*OX z;K<@{PHj;c+|(4p<<;rsF=fEeraLPGF|$~UW8IdnH~bc$TE4Y@g+w) zkBONC(@lCC4mo#*J5-W7n;P_R6o!vjFLDT?YN~MV$T?wazDf+s44yY1(bEAovch$vH(6rVzohv z{_D}TC-}7ybuu9;OcMhZk*0`km$MH}v%(eH1*3gAM%99@_1FD=>Ddg_)Q3;t7m zA|tD<-K3QG@7zzS{z;=_4^d|{+y&jcCH3xZa$|#**8@~3W?1|}x-j_`U z_Jx%N4~Z7h;y-iIV#~=eZlA>X1{3FrU6qKG(0Iq4Vdo&~vrPc(!_bzRnxE;EWd>Y= zz(Uiu0>46pw*NSZSe){y3QjQ$7Waz3`R1mr6iFQ~>PD#)%q6os>>n-#t}7VH%TwZFKS$5SkDg9S7H@T9oOTGwUO^RE z%m3_CI3qNsdaua1k4scFO8ErTtVa@a30FqXNBJ z(%Vg~5DBXCM3lZKn2(+iS?XT=>xmxV(!nP;DNw5A6kpnLXSmU5k<95Ivm@k1%9EmJ z<|Z#yW2B6tv31eZY*Kogkye~I08sN+{QNtn)Y9?%feqB~3|o8QnVA{eTGxBWw#SnA zNQ%RJY09bvJp22G{Ai(Y>Hak~X6C5fu3*PaVgG~a(9Bp;3bsD+vzUxiIb>Kc=!%F7D&?PxMl zLs~~ieV(2qle}8)b!JLRHqW?J4?R946x$8k3O}aq4pE_zyW0LZv1XHvjefhkgrc}A zI?ShIV?TaHYt ztiHK3S!~%DrH#$&Q!>bKCpcSpQq<*j70%=Hje6ubsh>B(;2$heq#e}@pOTznaGB(M zVK?A=033C^a4r8xG~pL7c~5T)th~+ltyKY()|o+~S^pL|m&np(1CK)>5W@~>^g$<& zG2BB`>-b(BN~WjiRUSh5y6OL1vNdyfJqh_uY zL;_!ifk=C6lb9465)3*tuV}`uX8=xAZVLI(@3FD^SW3lzEn!58m_9A2rJod58$RU{ z)Ex7C2Np8y?mES;_6==^hP%+x4`gcfjy1NyksvT8>wRk_hc5%%ISb1irkk-$QIe*n z2FE7}J0C?ByYB9WCMN;DGpkfv-yvm94tV*vJb46RVjmuW@pKDtK2BoK@YmwAybe2LKU`iH%tRh3j%EVdZj@oFJ0EqKN7|FD!N1HF%y>^y9m!tyB zAX5Mur4ARvq$<@>U1ktSC^C;UjZ=uF;A3r#^=sRU7V{nlC&B4K1r5fSFy3EnW#sf$ z(acKuad;u47hL4~-a?^W%YuBM#FbP+NZvG=KkI>%nc%nvJ%g{Or2+6603ghTywCTP ztB@w1O>;Od^s?dSiiLiw-RmtaShyrv$Ct~{Xp}isa;~jFFDTp9--B)V&F>M`NS4>vV5`P|uExycP&Xsi<3N0^ zziZ=KSkCi;FT0$R(w6vjM@N34H;YS&N4O_)t`|6vLk*?-n z>pW}QYsVIgX9QU9uGc2O0Dh$5RAeMO1aegtk1O^E7Z?Vwo~1vI9LjS1)%*`Ny^DUF zukre5FavmK3l*7gAV5-h}LQSm-n$zSa{3sWNT_Fh=|A$3oE%@vVx|IsYJp=8x?H4{zMDi z1zrv2LNds^ypIM9c$4~yG9ft9`o2w;E+fk-y@@ke#9@{;@?2zb5i^$vhSAxXnz_qM z+~I7sgXe*OCDG921$%*{?rYVw)V!%5S#Qb;v55zIq7*r9a3bm^Ex$dx`M-djiK zB2V`Ds6o*7(RoJ3obT&Z_J=>^nYCu41)UlW``2@k@4e1Ha6Vbs%f(BIcu2&R3&H%F z8H91UM)st0elq@jo;DdC_apAr_@wYW8ARdz$jGSfqX!SdGSazDj~ShE#92Y3saqVW zY+jl$prp)vu8W=M4W*~R55W_^qG7m)&t6kA?CHU8z~O;O{LJvrmhjwy)0EBs_Lh&G zD}dji;7C5bxk-FLRsb`5u0Ipnub7~jhHsz1Kq};<$+lYY>vVmjJ;%~zA4Dj5ef;db z0vqftXXUWK@ID&J-B>WHc+p9frruyRRW>4WI?;J%t9A*YCx`RW%jE$Sd+*oSYR|AR zEvKO%=gbIkZ2d6cmet8Nk4ijuk^xX5V>#?pZ`~8~X0HD4pS_sHIT5QG*MVeq0SCPb z9d-#6%wf;<<)5MlTHmnMY%KwFWlr^B0-SOk;04M_8BiJE{zCTd4v7wd1HA5!Wu_m-8Wg5|^otBviM8s2{Hb{emht+<`oBMS! zk3T-zxOV@$Fa?d0HWbgE-&H-v^GuqV$&y&=ns0uswW>3~Gk}ido7e_&8@UjM$-p)^ zy(}j?=bJaT{uMO5cpg07CEsoxUcdg{1vYo6WT)Rv;FX`aLKY}BCVr+eyb=zhcQxdU zE3hgiFu(dzZ2qn|e*eTxJe!T1`-5oxm*V0@D|eUqF9qMnqAGhfo=0kkMN*0ib2DFDKLJ(R81DFi^O&B8*3_($Mu$w9G^!-B=KG)jIdiyCDI+nySub)et=eit4B?^nl$8|*I7h^=(EBm@7Q2Y%HzxRvOv2AZ z#R4QLGMI5K`2QoN?cwE%wyVwi!hc&ky>PkT@x<#jZKhnb+tIj?X#V zOLlvn0$#YZSzCVLiAF;>X?;Vm>%(x@H_CF7D-G{@BJ0W2?#Y-?KAq{nq3vMwYn3Qg z(k-d}Vn7%3)@#o&kK{Yyf4Lt)l}MY()%dhuBLZb--~XqV;8ZRZAz+Pp|&e1RZyxKMkXTBwydP z5x*LwN}roi-aj6X3FX?vCfH54DgQP~58FNF4Op6(y5DV1o7&()c<<2L!j4g(?Ga(m zXWGoY*=;+7nrePkU#K?FY6!7Ov%W|q2M2XMk=E(7pjr_yMAuj63!xnsQpsF zC|xcNoRU4oswcC%eS+I`MeqnW50MfH_{n~(X)0X?=kZg&&$*_Tqxo17?~hZ};tzBz zkE_M}Z`-@O$BvKDWgOt#@I(dfE5tK|1GkKi?-O+1>*4H&9*uq_nf0~|ddVjj*Fjt< z;dyeZ5!4lG{IjLG)8f<603Qkwx3m3*e_ohp z>6A$>T1WuyQGaUgr}@6(yLWJ3q5vRMTi^6FH#MmJw{~<+@<1d6a75Jrb;rcW2CWQf z$bmu>gI|l+?7B*UZJvfBtK-qn;V(WS+o6mDNM#kNE(-lo2ckw7rNSJlLwPxy3gXf-qy$9 zA_jT@V~uvq&reyU+W8yBv~d}d;*4Kry3fDwvju+Ls+mqd|L>NDw2@MbO=hQmfHpu> zExUDM`qklBVF8sl2_R~eA|s2_Tm`l5!c9~C!_5DlA5~Oxffy&poxf@g5w-t=6oA5k z`_ICl=L<~=@bn^NxDZw!wKIc)0Q$ULZ2->a_d;2PqX1tXvBC)5sy5(e@*c#Ckoe5= z)R-^2Ih$iZX1d7_S@{MiCsMm?ZQsz$X8*pi_(#VMy}RBw^L4>d)E z>B*O%P69Nw28A*VywtTo)@SGB^zX8fcKP4Q%zVm4KJR_BCZK}xMzmr+n$p5#F=CO2 zV`G3cg>YQTzG)F{uhb$qzNn5O1gwlo*Uaf@-?T#V4e2c@h$i03>@0BkaE+U5Js+$09$W zv3Ei;CaFfb>Pn0vC131a_SMb*%VvyJDcWz5da5G$@A$*tKZH-2;DkooJE8zN@SI~o zGlZMvSApJ$q0e>?ZK_{7J~O+Rjgo%5`t{yz&cNK0;7F8V+xXT*ZLRxorhSw5BQ9~^ zZwEILFVu7>)862Vjgx``o#GNIz#O5lWs#*?n%~OKllaM!Bq-|Ju;g6Jf>DN$n%b5H zDh3aF_34wok53w4)PQb!s>*y*$gsTM8!L*N=ekN=St3u2;rD6H@x-FihwcD<14m1* z(_-skw|Tfto;NfuBiZZubMk@e?8(W|2rn#g0E_9RW^+fRj>K0_kSSPt)lF_G=Tsl6 zR!;B!W04UE?9)_Y#LrddjDu1Y71r0+Rt~F;gJR`+wI|Fl!nc)$$HB+8#G5xL0JM$6 zue^M(Z-PMJi|f;{p4AZch8!yCr3Q@>ZKXo1XdGEMsceWebEv5Y?@U>^rE`GE=aS|O z(;4@>hW_f%TgWImuBzlye~!Ac^;0a>%^0oX27kd(#PQj+j`hfD54IR>kM$s1@8Dom z0w~Lt+0Tu8h*rn!(e&ziSB#H`K?= z*PP(t?98;Z%IUjyb05Z_Tion*bA=pWp0cK3l`ia8Xd~0(lVnB!_|Q{syn z*onUF@>d^|fCn?c>^hO0=sYPj`oqs3)n(eq_xRAw*}281{$JUR2=y%!nG zDWJ+yaVLK_zbZT{w(-1H8L6kOJuxxfagvcm7WDJfj4JH4@H0fF$K;gkqe0IoyUL=={^UOE8V!9g~=6?eb*NZSnZPM$Fp z2x2re>PAyfzY{nE_W8RfUonjQeSs9h!*^z@P%3&{=zSXx+vt7f1BJ01Z9C1^%p?J; z`Ubn61oxLHPtQRZl%TVtj$Ajonm0|o*Hd~0GsaXbUU>wct1tQ)JU0>utu{B;LxOS* zGlw?3{)y67?#)dvEIzz7CA-s|kz~4*v=Nuq7FIgDP_Je)pwrI=BokKleV>ZCGQ21q zoU&_lB1nNlNe{idn6;AS%+*mh2Q$_Ju{~U1((?l}h10Uf0&CV<>RbN$3)8S$J_lA|qZgkx!i=SN1W z0XfF5%LR;UjeUusWOtNV`gXCiK<{VC?s3yXM><7riuA_?hKz7^MAwUEaL+V`_U6Ox zjt>yA$Q%&A6TFa~O{|xxj1(F?FLb5GWoCdv2xYbEPF*Qu@@5QniyGa1g#61IX8>+C z;>Q^xpU9lhPC}}PfCb7dbdeBQUfPIAjK978Q?u2nRF#`Dfagb6oQv#GcHGTQ9o9w5 zP@;_Cg9K7nnYG(`enT;hw3}$##B=*$#0ViqAgL`_bomuguwIV)07z! z6qK1WbKUOHs%b$uCunH&8_J!tzFJevH|v)z@|VeEI>M^`g6{{@SY=^jsuVX?3)4L+D^LD#WgMui5nuJ!VYV@NnO0N4>2wjo>Sn0=R zB#=iULMwdlt;ms}^Ku_4SI}U}VxaSQb9oZqANf7&X(W!}76hojqhvc&SK`d332gIw z2eggeu4O_lZOdmMsf#fR6Jwu52G{C5;7Xa!UAx)tI_N5;a-B z0j!}Nd2~4742S~P`EEU>lhQ&PJSO-(mf2tb(z;pt;*UKDWov3@!Rg!OPUl#w2OuT^ z`&x}gwvYyf^`j6M3`z*7+K`Z4oznT;jn7QbP*xcRx9=>X$(Hlt ziz7cB?`vwujp|L0wqqiZePju&DqIiiq5%^yH0Y@G<&m@i+~Vrk0lqFWm#QK#xJ+Vs7q(cmoUpUe`9jMxP)n&`nVri7{nBGc5hy`N@NLuEZFLPY@X`hmgHfb`j&O`srtC zBO81d#7aBjo_NU?j;L>*5Zu?^+_1B(_DHf@`-Tyk2R8R0L3=Agbl&5~ion@$t`<)eds434kpF9LnYJC(n&U+p?D$=ac_f0KYqN z9}&KE{HGGs#wp3C*Pc862bFI<^~wzet%dG@)1MZV_&{k4m3$A?-$)SwT)ef&hf1nL8~;QVk63|N4?e}pguc2T*~2F^=_2_W^&VSaL9YtHD_YruyZ!+Hwi`p7 z_h6{D>Q;=fnv%JvsXWEb#GvmQVkX$#XMiyzp$Q0pFpWseC~%S|yZWq&111fZ`LW5; zFZQHdIk{xiWgZ<&pNG1GnpZ$)4oc34Swf@cHLjDC35vD3d6)|wbV-76(EYETay{mE z734ISu&vz_e!H-Bf(aR_tq0}BhhaG&Gx@Bnm0I*tAStz5o3xTFHX&v}ey+;vhv@j^}3;O`9?Zh509-O7rRPi(74tdMukZKjnw z`yK`}5V1GLbgM){8NTw3;qUh`AOLB~5UliU{0bb^1km_zn9o3U9>Mgixb#Q=N)tIA zNRJ{IACo~~>z(E&urv$fV-paCrUzo`v12Ma`G!a_#TArJwwu5qE)6Apy!`wQ-Q1vv z{~$i-W66hF1`jAn(Vp2b3_=~+c3_yfcM(Eh8!9|9(&p)&vkW?1<8PV8WLWT6!EzK@ z&|1~i$0t{!4Ht5Dv3uI2N(oiPUnOV|EiX`1?hj^&a~aKR+#Grggd~K}4?s!Y-7ds~ z4`pO6LRGwo^X5UQBn;>#Cs8H#cc`da5Dr;S zhx>F&O7g`s1htjE4%7W_b#K2#k4K%Vf7d|E)N>darezL#N|RB>nCerjIGWN&VXeYx zV7vz+T*1iRYRgHG@(U=fN5|oni_1Gyfs#uWKFv70P-a&C_in(CXxQl1K2t;V`X8NX zDx(@D)~ZCfQf|_^TO3M|Q|0^Sx3^afg7EM`rOj;*;KCRl`Vkyqln^9^0s@nB#y)W2 zwvS1w`pGX}0xU3G1=Ro`z89s@bB{GfIwQ`n&V78Asp&=#NSG9vjIHm*wtkJcxF>pN zLjuoyy+7pDLStb8V+x}JXn}cuJMED8ntMD*QU)UNY3pJYHcc)}2m)cT=sv8IJ^U6J zn<_E{YL>|NJpi4{KG2WyJ`sMNYdFSS=muyq-N8)u7=_sbj%X{b=aT@aReduR>KQTF1ZE_XT)42aWKC_toJYwgGhF^frLf%;O< zglB(+Bl{jATb|7Ln}6Sve-4~nLre?bCY?V)C{WEu9Rgr->AB zU;sO-W@E8l{nhB=FUl_-@a~)MNgnXo))g>Rj%z~SX55>>rdHmNA zSlyapVi@w{c0w5;m>r7aA-C|e|qXv~4CE+ZP zMW#g7BpNWpne@f`ES`0Da|gAVjLew>LBzfnNtGmc!3dOaK@;1y(JhMzH4h1y0Tu{6 z$@jzJqQ6}jtx03fo{S&4d>yg*keLQnX1BRE8{kt+^WsnpJRC1d=Xu}WJQzsPwX}Zr zDwHrLuS;%q@@NClr|lsY3i)g-n@u4E_X>X~O96tN$W1uQH#cW~)#qr09=xv*dBTjO zI9F*E6$M`r!!;^d02Gm<+1c5Fc${EKf$he^G7Z>99#fO+62V~~nUpr>J;tBcSOEf) zao{uPm|H5aVxV$LCL`-MpS_#IC`IJsWq^AJO->Lc_h5QAo%P^#`ZEn}iFjB4{J3Xu z26Uix`)>vpL;Cx`Nn@n-@{PHpqmP{2ms3Lv-k0?z?X;9o=D6)&zji3Nenfv`*upFS zY0*Dsr?vkcC#c;&ENDlE5kjYsY;BcXRNNtX_OU~KB1fHriz^g6dL^uKdXL5h0y*=` z(&J!I7`oYIBGd5nJTKG7Ms#68fc*52=L2y{s2mB6h2OmNX%n$=WwfOpT`ceyWQ*1- zbEyr5upH3fc0VoGHZT<^?prq$WE4Jt<+YjoRM)7oY!CcMNRX1MJUy6goRx+b)Mmju z=P)#7x7=mpf#?18@%nn2-N}zZ;%g9foV>6g!P6R_fvZ2oD|FU=unInEAs5E5bl}~W zB^kgDr2VXH*tlz}$)KBjxaMzwR3KAI{_&#WGXnzZ%Mnzk4u=&7$&rI^`b6ONJFN8^)rtY6Iq&E;uR@IpY1yJ%3e@uE~0Xn?imNH8Vz zeJ`d$2&^*(58f^xPCpR!KRg_`J^NF$*DfhSX`!EbX72$DEJJaY_YZ-Z!==P}0`OJT z5f&ROZAE)~7{v*uD?5iFmio9-f($%KCpKZUz}yX?b{>#LC$` zyDq?-waUx+9@$aAa8@}1ds%c73=2*~o%pj+^Gy!PGnXj z8Xq)bmDe3}%080P74!4kzpzDXo4I-#8RfPES}-2K<+&f5MFjud5YY%cf39(lZu~#j zuN?oaZi8q-3rh_~s#_rMNA~jYcl>;kwXv`;efsz5^uk`TEnqQT$sm&Q$88HWF?;T< zn5)R(gsi+M)fs%0p=CGg_^t>DGrYi|51tYt5NU=6;>-P;fhZ}s40p<6jai6BZurFz z0K{!;ee*M8Xwq#_=sXlAWdB9xP~;Qf58)se3epoW`FDn!DQOIW~%UQd#<0JBlE}H=pa8=?QSu{$Rbi+C7iABX3kJBR@)J*gu`aC?J zPktyXiwKKxwd`FNnffL6+vEF8Zd@>9zL=D(zPZCL(BvhpD$N3)NeUFrCunH~lW;0=ku~!bvN+%TlWJaEAPY+3oEsJZ-Lxgf$@4kr^hw0N~WTv@~H9)_S_&s70<0|v>6wS?G9>6b2hQ>N>q7`BR6;FmtW7&Gd1HD)i%4ry*vUbkufzKDkEnAAs@_o8CBSQ-Qe9`eM6s8 zKd#seCTtK&>z03NJm_XuDH<>iB<|pGaCh%1&Ze_*ymq@5Ud%y~GgK^s>ZmwaEne8b z@qQFmV;f5!5G;VQCN^6DII_(WO29yj+#m_ayN;bVKw@Jw=dqd3LPXZ1o03voEGI!m z0=vT`YX&#$HwK&C&kt`6s-&|1Jd!|Cu89#7LW98XAu;~id(wVJ4V6Sl(f@$o)x zYCi#MfQ5xJlcOmQWm%>3Li4}yt}$Fln|CnsrEIUTc=mUK5`_*8kGh$!fEK*rQ)6bj zt8|h|r)iS*E6&&e)KzDEte{*B4 zEbYl<4~^hLwYRHVu_B#E^HUMePkm;~pD@4(1Z$g*am_eUiu7djAT##E^oa`Z9dW10 zlETf-)#`I#re{cpe}Nf5FSr^qAZpI|`&5)$?^lX%P%Ch!fLh3Yile`uSH)Bp3kxVh zQbI!=fPoK?MTVL-j24EWfjq;p`v`0J0bD(1!rxq_O_aB(hAd3p@4gL<+=3>hVzCZt-=xTklYhwxpbtXZg~ZE=sYJD-2LrGA|JXQ%mi5IZUq zqS0>hey0BK&kSe(vDw+aYL5Q@V2$v5hmcpdhWiF~W1WaXMC9@Zey2Eq%cnJh8gRzy z=<_i-5XS130)_TiLG3;c=Izx=q-tr>PN>V@jjqTVPm!OOLcvH_#3~@}uRjP6C@O{Y zWw6`#uKS)uayKm}{;6D|EK2B$U)?ZZz#k0JPtCq9e_o0hZ^Z^Q3|Hyn?%Ee3I5;

!PliwiF=_!D)(ijB=p8yT2+Rfz% zu!P=R;6C(k=y7_*0F&wu$>SgjlMdW>vm3LrwBq9CTyG*TA(n)rk^SIB096k!?|!9} zIH@cr7l+|<9p-!Z2}K`(wd30BbyVrIGt2;B^rKwlY~`qxiHFM~6O3$T;$@KXE2(0= zJTHTStY>G1Xvw13@pJdPXAOXnP z&%j(bT#!C|znE`|{V^k1oHBE_YgBF!Mi|R)yu0)T;??|23XX+6fISBy$;k^})K_q` z_H`X7kO=>Dh#4>55f$+bl*H+JB*lg-@Plq{Sfqa`-lvLZP8jU(mEQ|_gcr;Z4xUr; z`O&}_Am(#l7lJULjPmj2K3-r7IQHJ2oy%PKSx)fETTB#Wu!R?Qu3+N?$+4-dv|*tI ziDRuV0v8;bcfHg_z&l}f#jwesF%ClylqyTvrP6)%$Hzlg%T}sNcB`_;E!O$r^W{Vc1W=7A*2$i6K z0pS>^4nV@XM@xhYy=QS9Hbt-N)KMtCCJ%pC*9F7|N;q>5WDHMEKE{Bk)8_MQD*h)c zPh9bG85*m=W6ux3N9qWus!}97ut0X<{*8UZE-gm3_Ogc{KOW=*SGa;(qWD~x?{Hv&lkk}C?(&QZVs+CaT-dI0>xz4?Pe0bIuzt2 zFaapMX!t3EC#edM?K#A$(cQ0Q|Xt4_UxNWKzw|5QKpge zdsEG$WU*{U)>L}toA7f#vJSBXzhncp=ixFDhuJ@>9llOFhd-k27+Z6Dbf^E%xkbAE zD$p+Kvv9n=CKQdJ#zX}clxo*==>vfth1J_kn4$bKCLN^hFNshl*Fy@59b>bZoPl9) z(|Ag*D*jcc-Df=7nhp$wm_P$R=Q!OkzW_2wgyL?A&}*$5fBc^F)UwzZWw|IK_hvjl?{P+jy=gq|FeW} z8Z{XuI{k4vjh)@6qr)@CG}J#cp->X!O{HI(#;>y~VtBj`i zNis2xgVXG?>2G9cZb*pm-Il{O)cogOh}6w&ZKQLnkGa49eM$H`Q($y{_OiIL2Pg2C zU-dq_iPVIq?Q>PtWSzU7+5EpMe?WJ*$KviFP}4<9;PNG<>RKuh{KBG?veAK$-z zg|dms^C>Gi{{;%1u0vS`T5f)73JP-9>phH^Je&|f>bL@yT8Op-{AZ`RFrh5pNi|8P74Yb9CY>LNNjB_+F)5IUIj;eS@uQr4xTkx~AIv91Rc*7z16NB`~H~S|F&R z4F#dpA=*eib#nTL$J%sa5-T^0iMI5>q)E*_OO0*&I;?`N+-tn}2%ieVrUczpKgFB`p_Nd(6wJIJ<_e~@Fu zreq`w3_=It{BM9nBl>F_ET94dwyXme6~bu7jU&P5&-+5Y4>+-|9n?)taIkW1 zbi}%0%F}WW_M&(x8yw|O3?X=9D+%B$8|n$gHM7}8tPNQX*x}=wz=c=~G-YfHVWbqS zzrpA$UGP<%c7Q5FKa>gfVYqnzQE23S?uw?%y&X_=Tk{`#Zw0Jpz+HAYLrsl-w!ALmI0c0w8vW;wjG<+;xn5OYqRT3!h9qgQ99xKW zo3|Wkuzo&J6=8wsrf#K(T%xp9B(C z@E{P$%It&-w-G2_2xgkHGTa*}fd$4heJsxeQ~xs9+0b}_5RUzmIY+Q4UQP)`K%|#a zF1w=3AA+}fu=o?)dmoqsX!H58m0|5kg7vIy$wxISFZK7`1!s>$NH1@ z*G}3A^7j@IeT>&?3#Zxv`5*Q38v%_3sLNVHvbkt-&;!m-{oVTVy_(du^y^obE;P2td`h3p*!T%(=0SKGy#kpeeDU49 zR}tqz;N9vyAkR<+N4H@;W?(C`bb0dOIiIhbwf>!(MW8-o=N~>Q5T&G+t*wt44i0vp zv+#JM61}yP@YEg1_mepu+1Ex0l?5Zi4H%Xg@8genC^A z)Jb51D5-z?9sBxwdu6+202)}kzHZne!tV+Vwfr;*0I20>Jcz+58>dgeWZAFS+4s{G zlj2CB7P{(MQxOkBXg*o3Z(6&)q0Bk(<_JnrS=p{4>>3ViGhM3hNF7@{#09ZZ{%~9^?VZ&?RSYFI zFabsY!~bUcUv-qYJa(X2!q$dtBnUSVvrN8x`}X?6pLjW*${{4SCvjd)g0mD2aGm|+!uV5U`iXU%6vt#kpR57@!I4A z9~zLm;8QA8Q*u(c)pV|zYx zd{BM(@{g|Bd7iFu3&Y|f9dcpW)2B&6LQfOLLVxSZA=F?6s8P-g2K~%W38~T zPc~0)A>*y%Vrxaaq|@R}Xn%iM=MNq{CUE4qf0@m=&)Kh}w7TZBlrwO0wFi9r7{Geq zbK{rF$`ASi5OkM|(GL+(T^YV&> z4Rf11Zs1CsB-diKiKM4fg%l(DT1Gu(m}1RC#XW8`wRoCkuZb6xd*Jf*mjj63SYC6= z8Q8pB`#}XFxt2*K{#W?>OL6c&K%qZ%u8><=(q*e*Xl1I=a21`|x*2v}&%{oA@=ilEJVea&iUiK!o7~T@#*P>RMVhJZ-kN>VKB+ z{C2=cw_>MSJ-##VbHN|>OrmXd5`p+_!Am>`QHcirU=4%*YL3_?D+GH9nE#y%Lqi~~ zXDdJB{(H2W1StEYBqrmV@1MuHh2Oy`7Yo)(m~xFjbPnQ?uxW0!0+TUp*a^a#N)> zK;~8JtHE@KJtyqYDO@*iIM|2zDJ?$RSYt|&1_zQZ5jMEwp09M(>9v|kN_y$&_#$3~ zuTUDZ+_QRfKG?065SeMzR*jJ{=TpVP^BcY1tfXUA^GI~=i}S;84MKuXo<#3#D8r(# z!lWT>>WB)(zy^kbj~wO69mxIlC$G1G7w(gRjt!5)2T@(lxHcqkzXH7mgGP*MN_8=W zG`1dCQoy+@#EJ;~ZX-e&wL3x;V=fef3GvejJ2F!v^?0t?UVYhyjDbOsPxw3sBEdOdBRt-}>L-cYXOaL5kRkR3D>N{n+Z*c#C6(li z#QTGBWL=sS3pU8tL>PA>ELS~V8b)cEKO5__m>$Z&y{qp`P;3gz=kEb@yS~B2&KmFl z9w<~WUNSz$uXm=1RNP!s|F*YtH)vZV(>G5~Qaz!{PM*pD!M+*dk z=y4%s>K~188wXhv3IB+@C}~NqsG$(r4m17FG=Vwb`2v%=W?t9~uou3AD9A`^`{>X? zU9mAF*yDf@KD*S83Xaf)Ly;&lHtK`(&9Kv5QMqy2HC2KO+uL{U7t-!xNvVm^WaAgt z0d)ccU<6xi-tS~idqCy|A{*Yb3{4(ro9QZL7q?^|bX=Sy)OhXY_4jK66S9Vj7zlH0 zcn6U%(Zioo3Puir_|BChU?sWx;jWuwg+p}$1cL0F`T`=Xa-fQZL6(5!^xxv`)mULn zxULCjax0h#e>C;t?Zw}>Z1}Ai`d0pfmg9IS@^{u=cgJJfW)~#Qtd;5wZF&4VA@UV7rZKV5ulx3P z#O{IbN<<1PdKOr1P#GV4%uc-84UQ5zHnlJ@$Uil&?ozq&dtp0_XHJRco6w#;gh7BA zne{#p>%tUe0gli7@f}jNz&lSHB_QVit^C_tf$70;cSP4c*cl34O(@-bExG7XY>8h} z`(0TDg#eNMWdSOorh*Q7b)dx@sRRYDmvN$pp@(^Vtz}7~co`Sd4Ta4~Jdrb7; z0W{D%1`9xoV{@-_OWCa|!y)hTQ5d>?(r^lIp_uQsws3?yun{vqeaI{yWR*C$EBCKRH5Nc+^kq%w9VR2hZ|k!a~Q)9!!H`IG8zvw^hQVchH**X%<#QvT%Dz0+n3)!Vg>N zd^wO}z2ouO2FmaL+hqT5w!gvRi~_FkM=Tu4k{@d0nL4HH7|!j-rv~8lfKY5{+7xsD z&T`@jJ@9FGC#9ZfN|{a^LrW+Fmnn?TP_~uIg+SmgzL`Y#D=l5vxN&6nNg6>BC2k+J zhg7DD!n!qgHC-lp{P?aR(TvmN26H|HKZ=HkIp^t$M#d z&0Y*up{|!G(sZFZ>TB;mq(a)1E@qobMER zN!Tks6Ly^aKC5iPSZ%v)S9IBe=pRq!V!$0^VmiZV$#FVFRW@zSG&nU zW))wRd$J`KhRE5;3>?v=q&98OuSnPdfnbD%d3gg_h~{15<;ZkmLGeR-Cy+Yo;>nfx z873;ehC)9SCA?ACs#9v%SY7lH z#d2KQTsu6DZ5>>67jp$aek_FU_Re*%^zv#_#2%lL)~LVkjnh5dt!bQ>WL{v|lz<~x zybV~@IG5{)0NzvY+?0fcm-c?Fk4Id1+n5#oL9p^`gAi_qI^x&?5$6fx<~sL z6dd|y5cq$^o1Y{H1z2uv;e=p@>8Iy>^aYdb07d)ILxhD6N(|sTzZ>CcqA+B*64-sU zUDWwk?DYghJYgVr`n?45vw085s#blo>v-r5R1`p6A@%J7_rp!Qx0)TM?1QiS7n(j+ zAdtn~1cM1?X4b$V4wcn_auEmT?fN|nn_o?v_*N|l7n+>y_w1mD0h%{>=oyF%_qn|u z77=~j;56ca9R&En%g=&tAcIV&vVutE*0r8Q=!GxULtI;p1kI7vR{~y_zQv+>G*~>u zSMfj*uEe;nh){HOrWHE_={pQo|3Y)*>9U8PjVH9^w^x^C7YTFl{QT>3wLJd{mzKDK z8X~El+OfR-_u=6s)+2AuT7&awX9q)IQu*Q~-x~s?%s>JRfvrDzP!}KvGQxps z>*4AFRw@Fis26Ja6+_@X+Fp5sz^L5_VQU6ZB z`(r}21DM}z9X1w-g7NrD> zUfIRF1tfEGdw7D~wzFN%hce!ja$~2y9*pMVd3)nfv1sjPM%6uMPVkc~UsD6f067Mx zUU}^`yU+`8TdT7T@bSqax=t=bkwbboxN1gCYr(F_#515-6!dzhCyJ!_m{*gT)*TBWD^()%fDV-4s`$|i$(AkiK>_Vs$#0{g z&s1V(9t%;{G(YQ2naN#RkwD3w5E8X`p1BNG?s=HSx2}&OIlj5VO24cIdB#J^G4t&R%uAD8&k+CkAj#mURYY?{ixx$=Ihn zYi52`y&+g#xxh;^6Z|AjdgRP2OT0Yq_&_L$mnA z>&Nl|NaZPc$0Clx1Chbi4(@8hm359gsXxvH0`tVmm22gD$D*2B7X4fKD34s(3}r&T zvz#hbd27ToR($in-kz)-op-eHHlM4D1@j|PhW}*~y6jAG&VM#dlRuucp}(3^Fa?$i zfB(H-i%&13c621v6C2;O(q%-`{`0?mH{Lhm!J(uND4JUUXl-N#bH_m?ZzOVhK*qh8 zavrueVTaHYy3}$CMGwH$+Ej_vn)d()TA!CR6a$7f@gsvES$P%lqfakXpS@uLcEE=Z zlD@Lud$p~74JML$UU@ z$0x^amI3N+KsA58NnuRbNM>#-r}y4;_Tn-dOg9G#ut)y3m>tS&bKGm5 zXeaHH62*PV*%D<Zzf-|yc#J}&^T z&HN9N0#m0gd|=f5@Oqz@ZnX`hm=U~u=`SQp1m=Akr(R@0LVKo4fR)y z1{-okS+2@g)=Eks(8Xb;ZFnfNhLE^>zk&rihyW`O#5JDEyr7cD4SIlu^=KgW34SUw z5t_tUFXq7?ajJ`RkGI!bH@<}J8+J<8CHVgdn)l|YL)dziYt$6x@@Ezm3RAU};2f&% zTWa5RGcW$kbg%a8d>Fdz`~l2=?i&b&LbHmDR6{tA^kQD3ktCZ;upW|Kf#k_b72!E9 z?U#X_tuLy#w@>$0+LT&afJr&*L@rl*3W%UZ+~wHds4l)smFulEKJMN5IosOrmjUoj zU`FK7*;#$)=Tea$v{NUc{tK_VJ!qb#qun$WvT1!pM1Uz=i=2k#B`+i-68I>B#CgMS ze?QvVHEHWk|30{~-g>5!TvEWO#z3lKUa9^^-2lXoKD^?mda82&7#EthJ#uAgtyEdu zAySL|<$xTwn#0~5iKRDXY`CNEZ4tmXS0#;ASxw7Z4wZ7NJ-gphi+}NDx zk_)4u91)Cqa`o@GMZtg@PyI(U)CA>0uP*#}V8QVcHX^@0;zbRUY7ez9Z%DXyZ`|IR zJ8u;|v8Gj_kbM1=^6P!=op&I_MyB$wm)EJoV(U?71f|MkD@ahWpPEoT`YoF};tIRK zj45(bn^t#NFh{%adp6HqSThule!eA&Vmvd@|81QmN6m3GPF#8I_kSE+V|bn27Cdnp z+i7guY-2WNV_S`#oTx!#8;#xAjosL`?c~1SeV+bKPfqq;d##ySGhx*h&NMxbkx=-_ zcQdYN=FCCyzg7+iUE651R-Tnr6TUNd`E7|OeP#c0_H_9-0$5{m6po1LW`&g4YT&ed z=C9EBxK$*O!kN{kP@qAl!AH5B4wK&A$5I4MZt zi%{lW##S3b(O5l67u%c zPUF>(amqEtuEd(b1Al{NdwY{F?CJRN(&y00x9K{txhLsgPg{l+5yGlX%W2050_Y0B zsi&*XVI$E!h#WpNE`$X7Ef6_fB7%> z^&r+Fg?Ob43-T!t&&-hw|BZK0-n9ir24*^d+(uCref{D;)Mr;lX>~Gs@~tl zsvks3>j5(w`(~prVq!QT&^+=_|9{2_Qx#&YiDugA#T!01J8&eg9N@f%)p{;9I~96t zpb?JTh5+OnhS1FHHt1N5Synfwywum92EP#<-o3O{%nJ+18z54VwGGa&u^PH7ZI5=0 zS~nu009#(lOg)E^!B8kt-u_*v<|aY${Xjo@WPyNOsldCj1S~F2xCR45-_%^c-7pTD zL}m4FKwWT9gRG2UsQm+w*Z@W;nkDGaVCc2ZJj@h5rVhczuW@+;673n$C+D>5M9(&l zf{Q!n{?&W+u{sZ&OFD4AKS**npa#(b^kDYO0zA?&zd{9e_ea@fW!3&JEnl! z1!5xLY|nGuTM-L-rH&SBU4Xp%CoKm`Lzfv8Ep~b~BxJtyjFm7j5h%bg&hvWMU-)TP z1Ofz{mP+k<>FNb6+yisd!*5tXZbNDwEC?yf9$70Z1sMkL!viAVkDZBIJZd}3qyDQn zw{9p%MAp{F1VovSsxf~WK+trY1b~L^#H^0?zeef%;qmgu)MkJgRG#V82(H1clDA>B z(Gqj{-C184urYz3RSvAn_U(9HJUZDs`1T6sP2JWq3LH9jI9+bq%%DJ0JUyRw3JZv$ zb$*|ik#!=A1yq9M^tYOoqHa=G`~6_g7skOK4QE30-t(VteV`RDLtYs?=MZp*AB+7` zX7;Xq?w5K1qnZBALv{6Word4C^w{-`asnOMF-PVBlcCLVEMK#0mIL<|^L&H&*@0W(h*P;fwf zVW+4^(w?8Ox^vZG6PBTuNDYuG8itXC>1GWGAky}VVazg1yP1z#&-DlKo>09p)P)p7 zgqF8h)JZg%_%n};k5^P>7pV+Gfc}fJKz@cR@#cv7C+kTM4C;)d>_~)sPpakWOnSUD z94-Si0)>nQ)oTkTCXmzGPJJ`7o&jOZT>uw$@drxTK~b zOFcC$cw9K~ncIP_KBkV2Z^LW@HMTYe8wP-PnA`m+ppODre-J1}f+JRNw60S$7dU_` zJ))3(>~qKRjp+cf`zhKCSmbXx+w6IWAFHR;06Puo!28uEeZ#lCAQ#7(-g+V18DAQ zn@UTWUc=?7nQ4ar&yUT~UkuH{tRs*=+`IQ0Lv`IUPA0vC7GEG7#j;Q$5Yz^be{KCPdSHYdgc#ZADlrr!H z7gt6r#RCrS!5KWUt4I=Pwz8%6LVh152NS;BbD5FTLHV>`YQGNKaKt>0@VO(Om&>gt zRFW7Cw4u6yj#Z(V+)-TDCty`nDGZy8HBd&T{@lnLfCsfbVL=u3J?uT6UbL~@5?Hd^ zjH7_)ACIz%g>kI)`B)tWW(fr19^xfnJpU7DdHOi&KpECIH#c|U8Y6rKIF63Gl{yq} zP+(!>$Xx(vr~CHbJsNreqAI%;;a=9;KPy!-N(&TtBRq6k$&EQyH@Z?J*t^%)QD)i> zq?boa^?-?xr-(c%6cr0U9`N~uy^c#mf|2UmssLRMAUP`|cCXyjY4V$_=52wms9;BR_eu`}JZX9Jx z`}9H%;DR~>c6>^pT-i29MQ6WwpZQq?d_u#b%S!5DuPCqec6Nw11Jd+(zI&g&;R^-u z#+0fdRBB+4+~VEB1TmAyDe*jhg-0T*b;5xI@@Zc=p%^5;s=H2#!N=rN#sC0rKymPo zwi$t1Xz|an4^Iahw@*qdJqhbOv^p&(f-)O>}G2#5Q1s8UfT;5*+uOX*w z3M^*jAPn)YzG<5#hJIt?zaX9F^*G|@6|pYdDNsOHuo7WFDEfOBE6qXp75qKJmWZ!lYb^UoYA|oy|L4=$HQ&wNP$*>N%p4Rm zNU>x|jwSC_G)}P@Jt90n?<5!a9oRv@CjYj(Bi5Y>Vp=m~g-?~sz2H#hf!-#E8+!!e z4+6J7>K`_GoXnoGXr6@@irp{Q<8vRf88vEs+|&t^W{-hHH=U)lUjq*958TEZRqOU+ zwtojYm*%7$r2hTXIkQvO%(wsJY^U!FI+ zEj-nH8!mob7u6#wzx6c?TksN5p5A;D610g0(HEL!d?5a`7(xeG|6$6An;(h zey;AiGeJ+@WN>rM&1)3Ds|7o8VNKfo`Ob*b<=^FQyuIaGJ-YYUi<=EZyb&S{2R?~i zNySq@2&L_aqN>j=ue3=@vg^*nUfuws*ni=vPYc%8^f*hl*si(xRtw(V{vPi>2g|hR z-?n~S*x53=w`jCcz+}tENB(1D|Htv`iYrrCbNYoi^SCD@<~I8~w3P_SjdaV*)7btv zbGciifQn%U`)Byq&9O8&eSHSiPYGl|VCbyKh<+6d5DyBhY8K!i$PxNz0B#iX*3bSAnd}*#%i2);Iu5@pK^#^z_zl)Y%FTFHvV*CDX6tsSO)fUau(i~ zZIdl>glbHr5LM(TcEpAWC+s=`MM-_OdZW+JpY%VO`hUHavWz*EN7#iDm!>zBO-`CA zVR3i8HC=`*lBTAT(B9r^&gWO=L0V(2dbsmFxIOArzZ+#%Htdg6Yqaq3nYx&iv1v8J z1B=mRee#&j-4Ol2gFK291u+{kp!IoaaWN_z`XR|mK?Mo~2S0JsLwBPY^Q6T9?qQbp zu%hrz+LdkWdP)U}UVU31uUlbFu&^2jaoDtRh$>>t$B`_(EMJ`&4Vmf$b)9m@H{9H= zQ34Qmcz&eVJ{j*_nKk@)xqpj^IdA{*7>bKRBHS);=P*pD7z4w>hjzQ_Ddu#F0&{eHqpn+NEns1DiA+2Ek>+$MM>(}XE2@2Ry(q<2Go)TUWk>MJoraqIL z=JLoY#?6Nlj{4+k1(WZ?58z<0gQ&^qklw3z^ zjTtu`xviI>O_{pOxgw)Q3%Ke;KT*0MALn4~Tjbctf`lo~Q&X9{GSsP>u`j>79EeW* zhw894VpD=AK9)(#+HU+CgppoY)9U*r^s{C>Mbw;$p?rHs%c{yAL~@nRFCp`Kx;=CR z(RBC$FX{$qb{;9wO|67(#|k67h_5Q~pVNZ?QyU$f)2n8X@>=e)PATynw5Z2GXK5tc z)z%*}D!kFo9=J0Aa9G)Rf1=6B)Rlz7iPFX@dBa80<}$VWo|e8rNhVube6p^5IJ?#+ z!s~r6>F+NFjpI?K?Dm$h(j?OC$q{BJTlU=baja=^zOFFDUNj4|fK7&sLpyk)An8Ij z37y<-9f|pdkF%7nTkwr=8d;64iPaJN*IdXmNd6DaS8{pGEe~=Ze)O|R?a)3(Iv4&A z>(0n<;8x@OI(XpWdb^6iB+of`#FS-VEZVSq+U7`lQDJ$lx#8(LdZmtaYuooj5r@aZt>fa#Z6lawG?D+ z_9u$kbT^x9M)0ZE07$jCOKLeexyCdq2;2&pmwkNqfUQHUqx} zC}de6z?>ut$W8Y1ORz`8@+e~s=z6(Tl~_=+U9;xNm{|yz3Z!*Jfv>z5#-gp!Cg4PErlBY_G)ctq6H3xqtXt$38@il5CN)mNuLUHGn;ZTogu zP#^KNbN;NneALE6$!K}Ua~-o1J* zArePH#q514TCfW47(C6vcC~YJfJk1`tr}&7=NYLRMVTNX=zb2c`;O$boAd%OoT6EG z1~iTZhN!BB#05uQHWmjPeveyfoUO~g+DHu8IUIisF?dlkVI?)%zQ7<s}y#%T;P~It=Ab(RN=~$j*+*TUbR$Qw_?-8z52lc zQkc1ED%3XoevS1Fs2)<93*GF6q@%^Qf$$(MJf#?BttoOA4!F`)L<7}>$U+X-@hsShZ0KENO~kk|wbDLgfZ3AT z{u;vppN~s{=XyVXxDF`=ffU+RMnN&;x*-F|v*~g&Ns5};MTZQiD18oVEwp5#)AYuvdq(y2fSaLn|#ocwRv`I7JBh;eNgE#vOIuD~hMty_@QtRz|4iU-3O?+|-H|JCYdjN4yz4hY&_~Izn`;(unhSdltq~4UJ?V^elaxyeRP) zc6_au!iGniP)>E*Xc#2Xi`KipK*?@^22X`DIB%_5lu;vB?OG`XKYEG6VS@u2ssqxL zuX`0NWF>!ZeXW=*55bX#8`&!rrJF-4aIrnDjKZzHyQ@6>HCna8u!*IYEBgMp)#AeQ zej!|>(1(R4@qD+dnUWn5L6Rxd2ALcSlZ|H_kCn35=-*bHNT0qR^%dxfZVHZ`)6aAR z8)M6?-f!z%w8cS=j5a(OOhIv7%-_btU1J8|s1Mo|wZPbXU|q)w8=KM^t`rDXBx^j$ z_j4xid)uey1p@~wE0E!k_3pc+^H5It0DiKOf?!`gD5%Ggm&n&g1i%KzThq&h=J(mT z+X(i`L4jJ^ju#b8YPf8SZ9#jVQ_k_ORgj4B zVkw-YrD_ep0X5OMCfr!n^C{9-AGSZi+&@zOz&NyTdit9KV{%G}P*lH~zW(`DGaeWs zSCWb;SwKz&T%&<$lb*aJ^6?Y~OlChOg6CL&Zf!YLyCZ;%@Wkg=yh#819nP#-H9g_U zD%1^`wy!J9Tu>~y@LTPr`4CjH8dW*|38{Yfy?ey%bX-0Ap)6ag=Jt}Ah0tnrtDyLP z+8bI=Ox$AaVuqKN&hP&5Zb@3ROiZVo``_WsCma|#z@@_8GuG5B=0nN*x-E4uK{fy7 z)jALo-$=Ra*o(j3J<0w7kSfHe4a8NQcxGKwgt%*ahIuX_ySm*0QO_rrmmb1in(FHJ zBMg(aOaG7p2qg3Uozu>XZ+hS}8$1819sliB%u6r!zwAkypC^1;Y{M3%trKvc&tO<9 z6=71B*tbTdsEC+Smb0rRv#Dd@LC47-OlJ2T|D-mvCn_Jsa!)yuhC+1vXw>y_znR-mw@sfZ{o0g3^Ixt2*{JohbsQPxMQA-@1u58w z(v**Ew60a7f=(mKQPA&vy~8yoSj@`0K7_Fe6&3Z#=jjC72;Zehmobe6H4d$?!uWF- znc;OOyCghxV2+c0V!M$ieF~QY{1UQ1SgAp-sBG^hGZasG6ULHlpQ zf%)TOP4^2N_R`QxN+Y}hQ_A>#geOzp*OPM1kH@Ejgvft<+}s3XO9+cQXvu~U7ntit z+?ve|j@wtt@~Ua0YUkvhpC6ZO=S>Fh?bce!?n<^}=>zrVFS*n`|EgFVZ#-N)xVhf$ z4j-Gl8HTrvn9#e7U;QCNg4&nyyKiiC({J}+K!Z1X?itU>Kp|dMSKF0!d8s%D;-P~7 zRb5>@IIK1=ZICGL|8^aSoDmX!_w@EITG2P1oG)tx4(kyIb`-Lb!f?7q^CfLjS(U%n z!Nk%1{bKylNJilJBlu`hST!MHYj{~YVd&wihhDw2F!8xy8hDO6rxz9gVih>Vokgu) zS2AA`NKY%uT~=&|ePVnOcO#AxqW_g@oxTzBFuS{tmhm4xsLEcTph9`LJ=kZP<$zD@ zQB-Rey%(xuOWZXEDNF4ZdQ93QE*w z>f@tC3B#1!-18c0iIA)S6dtX&J`9hpH?KySmk(@loILY8Masc}Q_BToD(8$NphCtn z!H9csW6}|sw>WF@qPeqb#$1xTx=$FpvLRP?!w26 z=K8Ao2C3sJrU>Pns}Q|Rxt7`ATvv9CD~W=Ht;>(wm9DksLy(d!oDact zd7Ra_dE)poMd;uX0e-*bWqtuzXHK-o;ZQ`PQQI_j;MN;`S}aben4|WiF<8 z`G2p?bQ7Yy$@RKg^#Sr7nx9|2pr9zap65#3&v%9D>DYE-ACy$MED98Q>vzTe9)%+E z0BeAgG}PC4{%zugez@ZDN%n*D1gfP_A(rW%qX%5qfRde0>!|UI>ywoRc#HvDhQCY@ zk2)4?rcy>Y{bDj)#QrU%g}@#UTv3))P#_}g|8DC2(uxxZg8=lUk;<|Ok{IrBGZe&n9-I_WD*7pW zwXkN(ssA8FKHWgkMO(CniTUvqj2cwCsK#h=@p4;h_q&EC#e!gcI9$*MMR=zeZ75-zl4q7^S(Kh+tKk_Iu{%No^e=@x)-=tTrJ)WG#(VYbO zSzljDYwN^%Thp?47NKk3lxvoXj)x8|P?4yZaqUGme?Qvwah1MJX`Hs-Cn}7b^%fTgSIz<@ ziG<<#s;V1L4s=sHatj`#t=_}gT-VOtfMSG{qN)(CPhc@Fg_-L=g?&z|KV{@&Je6-D zMM=><;{k)9^O0yh2AVh?B&6h~XG$p7)joY}QE@8z=k;_o^^?QHr;M(r_oaHxhl`D? z%K{__GA$0CxyJ5+Xk;KKDZjQBHkEm-aaU^J6ZTd~vD5eH-!{p~otLhF07v7$@1sB^ zCZ?d+W5yk<`b4n|TsA@Ep)xPvU2$NMgD$=V_ic!H&*$Ds%Z(<`!+7AAF^3IgF~fI}!DK5<4vRH;!X(j%_T`&@iceLZpTWTV&(Gh= zeZ7|)5>SH#j~;ny{0kXrtAg64D8qfM1Ng?;d|%!--qvDS^qPg;AEM6MN6TQZn`hfO zksu@{8fagpp1HgWhieGFw|?Fi)KolJTx=5g5VRPy%wpZVSv$NC>GX6FgTn!im06tn z9dIFIC$mP`Ev-GQj|UGXp*&QIatq5!`1y@h&goe!!uo9eOfJ@Mn|Kg=BkWf2#5=wB z$G#E}P;4m64nhS2C#LA+d0HVO+nml)nSb}@ZTs3xV&=Jh{j&pX-=U%4drzgVfrztd zqdWP9vTVWd?s#CJHw6X0uQJciU!sPT4tHsuB@(o5tob!sB-`=ZQFy~JeXGW`fA3Dt zgoJ{h4uNg~0ESj;3t&KV$Yq3lV-}dQr^XS%M%wxdR5BVaDL+z0;0Mu?w`|RUe!4zA zggj%jmMxa~Kt&ewlMFyl6BBoAysQgUR`RHPYfCR4uFJRLAFM4#XyRpWY~@#Gbi<1* z-X8+Z>m8v-CSRlvvi=;VbVfZt_~)T4v$HAK;r=oBjgzUAmRfA6vlJ3WC7j3tBP7Ya zHK&9|&|%vX94F1*a^2sIC>T>wMzgZ)H@}>2ueh0y^!x^#ykwK5K0=;!n5|C=zwy=DWpv;8zx2S>Rncg8+o_hgPF;LVx44AF6)-Wq&d|Uh z_es-^Xt6#j0W(!;n>RkwU&dc=wjl&|qUKjYISJr2OQhT*JE>BpN#` zN8nr>t7UsYYz`_blZ`L5czmU&bLzV6-9jcjW`CYlbw4o(A@ubnS#Q0IHW~asCQB4! zySgaAb~ReAsaX!i*vC6&9%B~ku>N|Q%!E?^UsaDW(HJ@-AclCmt7I`Q?(LOQ&BX;lB2(QhNXcv_tKM$F%2$VHLGGWm&jHjh@}Y&8|>R^Od*o)hR@X6 zZ{SM0QA7mIGmZ^)9i2?G=1(_M4ud8#Tb?sEGBZ;zpWb?;O|u&-9C^9jYMyc$F03$T2h+u+MMP0vfRb{1X3lRv3N*{M72*pQQFFCk26?j+X;hB|5 z6>QDxUGD0nw7^T>bzf4d+f~l40F_)V=pBu=jnovzF7ZNA>r)7+d`x*zN(h?_PmDM$7H6iR_DDG%_>R_bp(RMVrvik zsxk~u;f!LHXW3~^e@gQ4>qS%i;vxH{aPmRcayRkffpplS$SNi1U?=U)#igf-B!;po z3m1Ff?;vp%+GobRPB_LC^|xi?i%PEOA4jD6#@6SJj(0D8 znc>p~xy%U>VW(c)h+9SZilk{LW>%yxb8fOnm%xTMXL_XX86L6k$3V2QU^YML$8+OC z_FC(lJ$Lo|Ye$*YgQV08HI2W=eT9F^0Bcl2+}_9Sy0`z+2m>6CZB67hCMitY;2>sQ z`%f43&1Rq1Dd-{Uyco3JTHA=eD*(ZP5;g!}x8v{NJdanPKW$V~v;F;B>wYP|jFE?* z%gx{Qwo)zy)(4d#m2YgHU=!A2iyWoBy(<(;|I9Gf%ft7Mx3a$8YHsNBtlTlQbpIti-Dl4C6CJJW^3TBh72PA zC0VP{>gk1u_r-TI&W3KUFpDMP#>S7=y_AEjSw}5KCT#4Tp3P-8!!dAS{vjAWy#E{p zmB#exw8I-0^HGFQEo*U&-2U2`A%%ujV%Fwc>nv+ES>%W zYGcFVLW|wciZXy%FlX<}ooYA62?-|THr$pFSj*=(Ikup$Rv?z<{Zh#Ws{Wl?QE6`dvy@hlMpK0|18YE zZ@Em)r6i6K?&|Hk3b(o03?3c^Og8;MNN0kQQO8yna(g)cO1q03R(*TRXitYPOX>Gx z3zF;SL6e=!k}U(>e;wPunVUBb3(4RfY~JmH!0y<|oRX6bv0GWC$*;5x#hKBz)H4hM z?&cdYbEaN?D}R*ESGC=U@non0a^Obp<(_G$CxyO!osa*y&*i}W_JO5AoAvWYb&juw zrV6BNto?EW`-_<}KLNbY%gto<{r&zC1Av%|@IM3psUM)aNFI@mq80Y^qswop*RY@%N={NF~?KV?`CskYC=Py==VS>y{`PC{pKyEFEwS<$V$0vA30e z)4i9#G6=D@65>I$K0pVEOkvJhVa|qOh0 z7Auw#T}W%e&u@>6q<4PwX~5VjvV2&27r4T-m%aCA41ef<|2{4{KL;S-4H3Q;<9PFK zpy7DB35la%Vmjdw7 zfW*nIt$7m=jx%hJW{unp<-p759eS-?I^Nb#{!!EHs-xy; zY%thYK_SK43%F8)@iIapn#@~xsx3A$VZLRABWh|GS+l z7F-wRyai|u@Ah@%M9@}uKtbVi;6OP%#0qplAp|hK)MBF*D9O2c<17f2yhwN*krrOIkdjf!Zs;R3QUG zi|WUW*o`OI;yvF>i6!Q5z(mc?^9*(T`Cq4pBOadr`|XWlZ4K(`@!y+II-91a{GMVj z`{-k!EqBzCpwJ4#3wgveozSnUUB7>jcy&@UcUtUpmH6dXkTC2;L-r4gXL8RtS`yRb zk>$c}3M6q81Fc5C)Ay0XCIQ>Gf|~=ls{%TFcP=Ly0C*@4P+3(DznlUSwhkcl>Uvn$V#QOWtaET= zA}v3b_IetF9Gt_#)$|k;98|}}Cf=yqV=?dqJkW=wsw0_+rAbxx zKo$LYB(qQ7q>dVNR_ltM?iOn79{kFtdvi0mU(}(?48QWo8Z~EL4N@iwiNtNKG9%tROxy z5b0@clp4#7)MY-c3Vsil#fY~ zm!-JLrhR#=lqh?=9uGZ*7fT!PG#~dp4~gF*H)rB>8gW9Qd$oc8!AO!a`PHtBn8E_U zY`i^*y!HWoxi91daH+DY}~>u~Mq{qnNE$VpOW51QmAel`>uf&?6_ zBvjzkPL?U%FE47$V7c9wqfgpOBcQL}YIK+cbDC)Rf>p*v?+*mO;MI2v0s)q>H1We@ z4Q9@qco!L*@lQHuhGT*kto7td@V_WMEiG+*R#R5g=V>wm1E+J|PTv41hqK8-d*G-+ zm(g52JkL@`FW(xLCnpFcGc1)dT3$hj?Zy)(`!hV50Faor+A?ETZ?>^KGPOMCmRHY_ zF3-%qJmo4xHVQNjAJ4(4@5cusB*@UtcV_^-=E<&*ls7;|6K?DXtKl}TzT13^2YJfHzEQ2p{Yl2Vqym?ePdX8e-q^g^52wTEekm=TZCNHx zBN_O+l0IrTzLzOcn23TJ3wWCXGjFR#n}HI$o0~eY)=yWj3~;YV&yAL_Pr@+7yr@9c z1p1VVNSYfg+Bho3dQV-uO5etmNA|t*sVObi-{bBr_+WGe>b8NwjV`Ah65+Q77hCz! zBlV~II(8&n@i24=_7D&FD~7`-uB!G)15_TGEl3T^pPZ;tIyx#=aVJq^U0vQjf-`rb zDRa*PKs40ipYdx_k$HRW3wuk}z2(%eGk5n}Tfl=SlH8^pp&kMWT#@SP@A_VHvY3(1 z4z3LgC;Imb5*QIV`puW?cPCfvdo(4?1H2R_RE$=i#pW9_-rgJxIt9@YDsSBn5=9yB zhtF2>IQRc%;f(em6MoiIX6)Nt#0)0`Bp@zDgaO4?@su{pg2uX_2PEzb33ISNkaV8G z!?R?YtVc4WC%!24Q$>L~VRX-x*Va>Scr4ciO^{p8jm?1r3O=l0p$bUiJLvo_7q~>k zS9j1INQ@H|L6B&TuasRCcw?@>{uycgGm$ML!%anwTN0!aAM|fJdj58&f|NGz%{=_+6AiA#f6e}DW0O8Bacg=()Xc8iM@${lo%#8YcufyMCF%)lDX*Ux&ZKVTcpR%Ej8Q@~!{| zAC1CIW~0%aoxem$o640S2P;)6G{9&nbLr1bVpe*^wp=o4dU z9^e&IHG4^}u%a6Z+9I_%WCaV}AAR8GwYH{A5q~Cv=<3U9MO<*|NM_K>*39wcmcC-i z_z4ib-LXEyK`G-bn63BJK;2s#Vxm=VSmx+*f0T9zsImNCRvaGgQ7Z4hdB>wm^w@IO zG9m)UZuhI!xG5VN3~@7>YOU7b&=7z@>*FBa{7r(AB|-K=iifZcmZ8mLvLVEr>}s{q zEbXgZCe}u|*l7NCc6DlRnM$mO72eqg^!70+2-IvAr6hH~Z%;4Q!NOMA46({e(8y#=}c{Mqil z1D-K1b!}U?t5_$ZrS%%|yE*Vbm6W*=HhJ1#kAn#`H}sHVpdcf=;}oHkuG4jqdhvLakAO)@ z)}+?Npm3>QoH!4(?hwqm@ZnH75W=Joi~`=4`}BI&729=M^HzR_#b}#39k>75t5|_S z)kGq6)LMa7Hn~p#vW8Lj+?6{eN# zW$NeiDam7P|AG_^5tZTOArIKzIw7RH$!8#=GO6oXs8RkZ zR-k3E;x3&U2zWL0?is_^W{gYRQv`~k6$;iPE<0e3wwe9Lf~8C}>}iT5e( zR`0z%by+y;yI7}!qnK#rb9HtrlAtzs2hW@F6g9W=WgfD^!ref3hO{3*;OF` zR-~~iq;V|z&P-mA0U^Y5SL|l4y7PWJO3K+APn9?8N9-U>02kssSmn!fq9{>7-fwU| z)0387aO#NuK0PYKbsFApC{Zh`Z|dr4ON=VP$93tyzQ{)gq3FdG=KDCGBo|xxzrNl8 z`_kYU8VWFH-m`@zR@^FTb70TbI zgjuXbCf&?A2g$_PlCK(^AW=4Lx@jFhT|#~m)s>z2rCquTxtZFC3n2>i69#qSf>@}v zHa|$$y@{o73Wt-`XTSsy8qe16$*^2q7*fu5^ z!|M#z%9;QBc9gEQwRtftZ!(j6n;y!ELHqkL0E)C-5C&s=pt)1q9FO{ z8R^BU(N&lSa~d1toss?1$P*y^mgmnpIk9I?R=+P#ac;3>T%Cf?tJc4G<&!gy*qqyk~)2ycTd1 zc+UWfB1}}gqe=?;3n7v#G!?=T7HGzlKU90#s*C!QGU1D#zobUyf6yp4^Mbhdh)bqA zR*G-nC8O@%m2Fv>m&s(dU1MX*vcj(&ZViol&yMFv=)5aNT}>4g+onl*n_~0l?YOof zt)@Begp-e`dH=sSxn}zuqN5ce7I6V>qNt(}FsV*Kn*l0jO|T0D)@Y_XI@5 z5$^z6Ikjx0!IusoFanG2%RI(gLeQvnB?CnFjUI}SCZJ+~g_Y#7v)=WbP*FE6rH@2? z?t3%weP55Ny`P+3j~WxNp!;1o8XuEDfvXgwXDaGfezb>c9m5N)+tD#Xwd|RUiC@;f~$l9)jp(1pF6 z#a0^wa`(}UD_ey7*UAOVxBjH!an+3~j4{pibsCN6fGhu=6elFAfb zq2kXw6AV!iYLFIB9szLa zf!2l$mmVSepeY`P7zTu&#ywJ6Oq;1n2s!~m^~E=uS(V)yn*^J1+uc(YLe`LssD*|c z%3)7-s0NTAk3bYrNN^xpT;_K}TY9ES$Cmk>)9$#9kEJXDxq-s*xQVP5fYKN{lJ6SF z=S{YwXZaYYH5ue_m^iag9Pb@%-=KYz`J@q^SV-!su3L z->ZfqQ zR#t$X9vOPM&dL-VE6hLYwA%HW(glb~zZ_%R=sPu}cKYwU8T?f=-{^#Rj#`iPpMG(Q zuH0@Z3r3D9L#}#`u7aIYzJT?TG{NLsinOQ85My1i!eSU6F1`dI!si5loA?CAPX~pismg7f{Jt!lP z-EgoXsfuqF>aPABNthn-pa$ZSlsGnkt4A0l3VoQ}D769-_(!ai3{k4~%G?dN3i*nr zeV#U+p$k}of0yt@cFt{SnLW~5ojI8rhz1H6?Av>TE{w|6Pwo|-9MAp5VukXKlom=Z z2M=_85eXaXW5+bKZa$vP8#+paBdAqQ(XPn8*swoU6s~=EsIUNN4>YyiPRLYL8T1S@ zR`cfUXJyr09Ep`~h1;oqN_(NE$$0Q>LR=f$|AjcpC+_|$mdM~Q^nUtXV1G^_U| z;RwJ^SKHAAQV(N?-OcIktbRN8xdJq2ldMF9A7Tg9pi^4wCGB<@?pd4lKb_J#*kU=X ztNAyh$Sgj-JA?{VI&$j2O2@ECE+v-3W~VUKvD<$SHQFrfM@Amml3H=*$}?GXybY6( z?1o6QdzQAf+4!{A*3Z~;XTTU)=&wyYdj;+Q@Oc?(y$Y;q6Wb#W^S%r;t{vEf2c_fk=xVB$?PQ@EOOf+&5X%3^3agVc^iX^N{&WiQjw7O7{Ryb z_Sc3VOSkNA&%rA6<<o91 zha=*YO*8c4$ut*jHOk%i(cuz+$_>eBC}{jP99D7D7hpe~*MU3sTQ*G=WaGkYd;S#8^Wqs2qV;2#;Z}wRHEx5nCTX#{Tg{ z&^htXFn&C?NyV(<6X*z}HS~DA3X~`v)UuC2Dl2~~b5M*3g;WxXEJl9zIhsYH4A4ds zEdNIMB}H9d13$2SiP6kVf-rmv6W8nH#5&tn7n=;qUnNQ+}$XW?!Uu>is7h#w(ia;}GX3u^^WzeFN=@VbqU*80xUSCa0O|=1b`c~9*E};3#1QwX^3eM0k*3q% z2=#DPEjBIi79RCDVvkiVyiI9(z7gJ5LyC|~I7$I7=0<<%oG=Jx9;Z)L_2XfXq@G(uBZpPEHfNaR~ z`>Z0O7I~bDSTgT@3Z0h6u&!b`#075hl++3J# zcfQi%-OMn$5!i^*2E;D9wsa@{OAm3uFl?fK!@R!=O|rauJh<9z8XUB>dS~*~@GGau!20Al4G!vN0>NRYHPb?8<_{})q^WKMffh%JthPfg ziEE6NH2NY$k&_N#>_{jK@DwF#rF!a>uvp_a3lmgkZQ;ptQ&VrVvov-v2oywoi`rTx zYuoYoxrsyHbgElekoRu}|MvE-1_uMpT^U0|;6+-z+Syt@X2WT2&0Q`*)?-E#R-J#y z>2k)Gf{HNWcNt5e5A^#a2*ibaF7Qq`AW7RS5Vg&0MeGt&#y}2r@9)76T3!Bz5kxu}RtKg(Dh@2T zy5KSwsyZpor^_z;x^xy?tbTS1ffEH-`H5VUhAg#ASy|@sAM$x{Kb$7bmkd7%y9u(ka_huNw-l4~o6$B{iy7hQ#kWAb%gS3&WeY%RvR4!KcV7{PcIj4EyXynqZOF2+F{H1l z*7T<6+>M}HI*)iQjD$$r;Q;X5&EWMeN}WVvoN zVyf#@o9Sry4!7gQCR_W&$XWn)Dqdr@z(6>5Sf?3LqJ0S~aqmd^M z3#Iz^bN->WUtY`rEw^O*iT$c(ee2*lyx7tmQ06ZG@isl6nilcAT(LycVYUE4&p%jc zF6ikyelM&Ic=u>nX{<22sZKmDtn8A?x3{q-;m5tnCGB((T^5o-ED6Sp9Pj!`akznE z73$1yR|bV7hB9OxFrbo3Hfk^v#55p9&y9iTjO|X{U{EBs4pS+me2x4~PQ>#ttvij0L5_ z?efApRrU)RV3G&MS7qY#+Dp4*jfLKM!gJ=g{XDW#h?2&>d1e8gHSaH*Nk5=)to$+- zk$(r_qeCl0qL!$4gJyO6rlIX%#=wVva*H?u_k#81v0LET#ZYPqY9UV|gQ;Tkci=oW z^eAAen!-<9;rFA*)xh}xmBRygiP+l*5T+ocQ45e_k7qAtnt+5n9TvG+_Hq05lu8KW z2-C0wxJejfy%yE9DN|e30zBMqUmk}qKDRHXCcX6hUF9PcZkgY%g}^h_Rja)3F(>qM ztbMG0W*XY?n+KTsNr}K>7@V0|PGRz-mP3W5 zN~CUIpXv|y1O|pgj@$TC1Bm~jAHGjopQg!+Z+n7r%5HyQ)8J0ur1l@*{+QEyE}Gh0 zL%X-;4klNVoL?|9K#)3iIc2IO`RuhlGLVbn(zImi*R!7qUjsUU%e8{@5w;Oi?& zCuApxMUP;`Rnyl#H8GgJ9zQVae>_Y+~^3-z@I4sCy zOScELWpR#2mduSGG$O_~0|S}?Ov8Z``AEwMzd`54GK!EE#1@wgrL580xmY0mAh`I^ z$)$Dt?yg8iyADcEan5u;uIl=FCu2!Bs9_NsEAl%>Ikbs_ytr%1@P=Bi&_wc_m+;EU z%I2z2a?+q!#X^U*x~ZvY3?N7^fcc@K5iz}yFXiM{cXZ@>HR)?5KAC}^i5W$__>2j{ z!g^!Uc9_m;8L2;uA@amq^7_{EI-2Ku$Mi+l!dwB)(Y99G^XKlWFp87N2o$WdSKhA?fy7sG;*NcJ%7+Za8hq4_^oth@g%z#e z6PW-gcYG`oj9+dcYm@qm512zC(q|sTNGbnn%IOZsqZqKR1LEU1z*I{80#jm5%Qm8) z0>eBq;tXS(X+WeD(+L!1j0L!9KjSK5gCjG3ZG7Uf*{;bt01l|up+2AHLQSvTMVi#ut5x- z9E8*XpA0n^nC=|TwDp1;%U5J4vgk>0`IOv9x2ruw8m2~=^aDU6$96~U$AyO%aPiO{ zeVNoH*fSNoeE#YwnC~yPogmgrv7*rDgR$Yk+JU@cx~a|4;E*7AcGTaX8j!$;$Z^VI zM;vl!ub?;(=%h$p0b3R%*~7mD`x%BX)v5_GSG4y=;DUL$zIdu>XO_jqz|`70#i#6s zt_EKtMqZwLC=*6&uUEKocJLBN%3+5L{OgcscaUW)cKa+o@}2mi-UT-SJyDh#7$RNY zuacpn9t=@ftp18&>wZymcNZxBornBZa{)#LRpZ!bW#|3^(?)}p}|xWRz7 zyJZ5ZhE{4y8eRjlt%~F#j zZq2QI)MtV!fUa&PGt_taxlgPH|7D5FmUMx2;A<~^zwCDpwpQwW)?*_U&X0v}O?I&V zWT|ZEj9?l}r+ek<-_`#Sbb?_Igc6$#Ug*Ha{ti_HWN%^3GPM)uLjH;gfwu5fU^E~H za0>ReU>tndm!4KQ2Dvi4O(^1?8!>f@>jj9h(1~zul?(Jhx36%)1D_3ZO)#N7^q<@8}EmGr$&CYW@Mt$-1-Tf$|d=E_0Di-XTGIdiI z%4W$(&!doSZ;jy$YP%v0eWIT=o&WAL1JRnYZK>x|S#N{bb$@E&YzzTc8vfkRIW49p zgOjs#{-k1NN>D@PT~+p!&)+Ky1j#2^O2b)vyx?9}fn?sp)pH;tGh_=3(*|9vqTVIu&;pBYBx(dF z6$myHgEKXTdDhI=cZIJbQvk;Sb2supfNGgX!ZT6h zv`GL3LX8EQ^g!2wU=n-S*>+hW5Awwbsx1a7R#KoKJR` zN3fZPqd3&$4mx>*2b;rOsS8Rp6I{p?7UCne9)kT1--b&bi){E?A5UU%Q-C1WoX(8K zF|TF|OkB*i?twUqS0CrE5*K~-RzfA(VTjG%c4h=cS=$ewoWsxoE&(!1%s*f2<48BB zb`qdG0@7>Xqc9Jby&dxlsttK1`U>$YPW>;q$#Z6EiPS4*n1h(%Wph;jioAWFnn@%V zg+E_qP0}fI{w2H3>*8JcY~ta_@<+wC1{;54EKB{5BX58I?&j+`n}yFPbMVIH%*rfN zk+Y*9k@|YPU&2gI0S^-d?8Mv9T9WRm2cpK5lhfOg`TF+M4@{}CrGp5=3XaEFi5_dA zD*T`knH7=oVafwFW4d*|t3sa}|3DpFbWab~Fk*2bi4T;|2eX4;1Ib!o1{xf`hC$CD zVna3ce9T8>uPVn*4ipkI+Lqg9*dm@mBL)-VC4IAihnwjD@3Wc6VAMOQei1QA|=>Wr7GC%7-e zU>-53TB2GgLUu?tB$X(P^N=qy3N;#RD2AX#S1blM6y&O-JX;L!gg6t3JK=wXfsR%= zZ8Cnbbt#w=AV-!p+E~%GVQv^U$OVMiPjzf|!gvsX;c7X->{OjwNF?TWhM;tA3{LH~ zLA6p9GfU`LW3mkbul5YbuVy>q3L|#lXx(0J=SzbDIP6EP+wmeve2%%+MuD<|dJg<7 z25iB&G)Z=0`)%w?04005gOp!L=i#IPR!$ zjXe$T!r>3mdvGNQWc{6jfaYt3xR z|JMkAs2fn0>HTcm5a9Ljak`XMP;l^EV~m`3SmBg`!~8ZjhA5X}5153&LXtuBGFy%Kjf;jYI4T z`t!;fPd05h+||D|fIwo;B3qX`MS#Xp=M$r~R-np@Di8kQ@V63|w6ea*6;gs!cH;5x z!u?87l2eo}so~?lv?}(W4$(BV)07k+S6ACVJRhBHZ7l;0OLz*>xQg=fHIt0gkm}-wt3=$c*E6@6O zxV&%7+0!cDepg0v-01S#1^NXJ>nE3oJGmXy^5nQEF{5&x&(cy_Ij2h(_qRWYivRPb zd?y&SvyVX(tnImupd^r5gVOKial-(R869TcB!*NE>oaNo<+YBodKVa%LaiA7nFh)) z+~918K@XSMUF0T`c|KGN^yB<|n`sY&80e&=T@SjQgqBIMJ2+Un7Oz(eacP$_K2xe_ z0L4KGHf?%mv8M+*3hl!`--7%ZDj8HDH%b6oll^GizO3%E+Tssz7TZseh5V0g6-&8-{%9?tg>7gQuV+D z!enX*pNdbh>K7y zi#SAb(g;Iy69q+W1^>No6NnMmU#RGCFMjEki?|LJ^l(^EG`So(E#LoLMcvx!&GBfz zlM#KMT-FH-)da;2>+0&>pHxg|^E>|pu$eoPFJ@lsg1kNrMbho}*7{rHjZX%BkwC@2 zG7!r8jqmt@_7&vt?*f>R^R=9GcU|3SPv)6P6PK;%&8}s zvyD)1t`N#)ATTCIIjTr*m&|HEE6ZY$D9d`W&KyCT)IU3dElD-|#3e35+FVFqr$gMc zwRL%S%9khVVV2;6p-7GlIulNl13>j!O4g!ds%zh-VJ^({RNq8uz;m_2qVFPLFACQ{ z{yQyM_$3RbjUuHic=rPp_cX!EjLOocXp2`C@I2D62dZwl20{@oip4rSUy8l<4~h{f z4%Z&XU1lSV8<6RL;h#9z&Ej*cten`?_Ue3dTK=N8V=TS#M+K4G*Ha3~l08dO(-37+ z1R+I?S>O!r-$)s4Du0#xVD`cQU|eFxPsaR~h&j#I{bVkG{LuZbkjIvOke`gMrm^?@ zAG{0XW__ z8y~-o)|*$gutRgh&yf^;pYDa(AmZ_BW?PIwm(rW9Fcdb2%?%FD>zw&g87L;sht^np zxIclM`BqInoMf24*)bU#0qzDm)wq&2Ngzt#H{@#wM6&0wr$G#bymI8H26zaMC9n?t z^YV0N*KLa57ncjy{vmSDV1zJyR-yLzIsKRQ%2qy3%)0NTAKvT=P-2M%yzbBXN91D7 zA9#3ELsF9jp7DTKUh-FjvlnrA03dtcdpdkiYu?O#Z8P4Zq9%k4>q2h>AN2k0ZfUgg zm^yf_ApiZn<^A-0xKQG2?q;*&L!zM5lw%__b~@+-iWv<#9E|m7 zx_n!6`3Fl~ID*3LN-Z1Ee<{+cf5_JGYaR6@FjI8UQh!-;Oj?uG*{{=sxYlk~os` zUv36mMQwWDugJ$8in-nZsiA5yfvvmnwzYC4rapX#-w3~Np;%DUAiEPPz53*DldeS<{R)#C)#>^-y4+0Rr zd6x#Wuc?!b9$JV&yVn?TIpxHI`vK^ZpzxKB(t_Of4}3Mh{T6*{%CuKS5Zrk&Up ziT&yhzK80=r0qKKKVJ!WcecG!3m!zoaUSGQ6>F*yy>(a6`ukB5?7r zG~qsJkTG(!y0>uxu`Egck50O}3T@2^8dqWx@C(Q^*h>w(5xw%&aKO`uGYGI8IqvR4 zr>76Ryi@+#TP?sZ!V(jwuC8!S=bp*oqrro=+}w>M{iUNLEjCpT}n{C|p zmQaIx;8nUP`J}mKpa+6v!_QRFyop9qKzp0)0w; z;XAgSA%WFg+sla;Y85N=OpxMPd2PvxrNTTKK)a?VpPW z4>+jJ%~J{=*`0s3v>c5j1P6)D*fhaI$l~aLuezHlHh=qhYuB$&m`YsHz~?Mxr^oT4 z#ikhz*X>O{{5cLbgc~PRCgM*$B~Eu9^5iG3>9~dzhZX*8n1_sy$!~h?YN}%U-#2*X zem#JvaB9s4NN5{deZeHhd&^J4K9$u!_D1fjNC&<4_)PGN29yXON;RBy#g+=RbfxNP zzwB3c+ywb!Z@kuI;z%Lral?-5Ha;&=v7{nEn7O#|(-7Q15R+JJjwkLQEU}l03b(q> zL^5on*jlLfY0}+qrux+{D^0-0Hr}sYV`OFGdVba$Ot5f}?Zw3`IJEQfa=jCmafD=L zRMdc2b+VnsSEIa`X5YH)uccPatM>Z8xD8ufe4L!@58*uDV znjWkZt>&s4Q`Ir2w>tFIUfE|YWMSag`ONYR9Y`i>pwtdD`&x1$oV9y{&YmIWC(a~r zXJN>n3A-`%AmheodE8_v(vNm(5VIxuRx-c2o zpZ?`_%F`b^<|DY{dJGA~r6j<>-Gr zk^ptl{hm;6102&gU$O#b=jYQDS?jtRM18I%LfiVoqsmX4N^Dqj)5|_oE+upH(qlzf zm!FUky%vc+A8fwAHurDC2j1VKf`DY7F}?*xhGp}rZYoM>s`*qkgrhP}f=c+xG@Hc=V~5Br zr=X4e09AkbHQYoUNR;H6SM#Am*?r5-VdL!R=tx!D0K9Jq#~H%Ul6r7`4V_t1K#+cWDNe(zS7hiE;_Smz z+rCFnu>44BR&Onc6+^@-lof{hTVahi%H9I3f-V)_FY@7of&MDIn5Mb3sv0G_*r!jM z6*XOtE0(46t!v0>Bq72EHbt;*B*2?- z-0jYqnxc!!>>N}-EJTA9=uDBTJxMo(w%hB0H|X&ju^N-YK9u8F3~ z+4N?*?3lk7gvIUc-Q59F`j4%4O?qn%rQ~~Fs`TB0@gQ^ML)&&w(Ps0gRf`5TgVTs_ zoDd9o{;udyypdm|LrKz*?}~>dHR9Yy;sgD%{COp#vkNA|*oXrr-u}YWxgL~((BGL- z_zaQGpsT?J+&}V>yil#c(9r1ZQQIO`?2@vsa`8t)X9;i|(PtzOQeK{apL97Cv1cG z{IiNJEVqHKjWo#^Z;~449Y2GSqpUEvpc0Ic+V%FsL6AATBfP_l;rr#BgOE>UApvljV4G3E}YE}Sa_ zL3EvvenQzX6+@}d7GSq@)nZeFkoW^FOQnq>U0a$77D$)v*sH2ea@xQWJ@MSHugGyX zS4O|3Hvo?b2sv!Rew{rYNcBke)VnG+fBih;E`npr0$*aqy|}ivJU=f3ztZV*7>bmp zlVD%T=;J=Ec2^^Sdo>v5d6pjcrz&iwqg##81 z1Gw~(W6lQV=GLkSOn0TMWkaUwddRW~78H`q1SkU-`E9x8Wnp&l<}`ps^ZMYW1HMbD zhv;GH4--r-8^e{CFl+81x6(8>`Y7x2FC2ULk?n|(Pm4H{GbT1B%)A7eyQsbQ#Y1>d zYL96-WcKm$pNC?zweK3alTSDpIF7zNLBaPpGNNm7LOUe zyPqQzaM}Zovp?<$fcYa5{X&eVzrv~kwa~+U`4lM>QKidmgi@-gMvlOPDi{2jo?I1> z+CqlLyl)VDI08Z1K%edl3q&g)c^S+tVp78<*&d!y_0vIu0L}wNPCPZp_=oNFRDH5J zfv(|Hdz>GeoMpmVKKp^Uy=~?YI~vPbNeM(#69)&!9Mt;t)Z;=*bhS^4DG92u8wO{G z5^JI@sF)Ee(JDJ;sH{gl&9J7%3~o50^2dkqgW-q-iUH#~U!X6BchsPa!^0`O$jkj( zh{O5W!1iN|J~M8`!p@C32P0=7{1%kf0G64z$smdyoY&R|_BV1~{V!Vrwy?aQX|Tf5 zmcawMgjcHjG~usS`(N)@lYH9kU!J6)wP8MSoF#n+#k)x_;(#V6-GL4;z@?Aeo!Sri zK5&=L-vfdi&&w^=|A=n~=G({oOy!2S{%M{N^SQi+@kAsnKD{**qV+qPRLb!FHpk6h zI?Nx~*)!*s!;EDUE6b5s-$v@0$IBNYCv!{QAA-6&`2BAhwiibSG(LGbF@N~#aQ0`h z5Jf0bKfCa+K2f&;nX}GwG0M-8r_b!fwoVnIW5vls|2pbDbs1LT64Kz}=1smjoGvas zoXN>RkEvXk-m)0jNH6tkR|FaMSW$?%`JqeJlO-j=YkxqA8>2%FUzy;DPJGu%|2mT?kHCF*^dl76g+^!a1uSAEU> z=jT738(%$bY~)onmxByv^0?YsBLeykHz|jkGkGHuKR9}$$;ti}d?@8yL#4$Bi`nC< zWFivNM9R}bmEoxi#?w+ayl(F%qU$gphpjX)t7~fVaB@C-jf#iGQ?_^Z?SJEbUr@qK zwBg8U_q-4)RSw0B_^qw2r4<|d;dTh9(LYjSNt~S8TG}9xe(=q2x|7d*%_bwkhdQrG zM$Hz7_J9xtdXOXIi=pGgAd6*ciVi{SK(QZiD4pyj_*0#jHLvkWEyDN7dEJbti=i&y zyxaSsXmHU2BO~W>g!p2jqkTI4O;tO8gkj0kE8pEE*Hd^^cFY&Trm26&fex|6u|nV} zBgl9a)YRM$xVQWQ^x41Io}%Qjfyi;AHal0jczH>vlo?_^?vCnOjN99<+uJ{&5ieWw zW`T&Sm;G8yntzOzeq>$0-*j~Qjh$sshNh?a7QdJ+1uKqne9kc(IJb84HOz8Uts(r0 zmQApBd3#w^Tf0kxHx$D1O^S?mbMQ{#DEFTSLM`}cv~cefE4alY2}(UAz9lN*)r$3~ zXLuHcid7DQMn;{GHY6lOQG;o~pbA1_Z`~@y$%%ouf5#6`++0;M#2g7Y>UzC8$YIDN z0-*w-@L|G0FjkPrpual8`}bUpoe@wpP6u1U@i>gvepe2 z9osGwWoY7Gl^rW->-$iJZQ6%9lMnH~2A!H*d4d@uHMZ29Q?~eR8fqo27l=)KgPfQR zsu_jZKr`Gd*ofS`%9i7o9q7nm?`P9*@w+Xu!Y8{guem`{R~6+28-q@sWm>xoJ0*Rw zqXWAc8H((Mt!KY2D9{Q=qA}4BM1NWsg3!?3+|g?6}58z)@lB^cE}`yd5%vLO+@e5AC!!Qczu; zyRz!lnN(an&4Fx+KXfq3!o|a)_+KLW(tFAERR_rl1j>i%g6mRx6+@Q=vF0Q4L+^EJ zNd&=QME;dXQ1WNKU~RrccC@rFq!?*)F~6S5m}Odx9AMmWPkdVGttYW&h{NlDrNC~5@HLL zm<8gZ7L~d@O6}fFRTZ}6cb3$c?Y>XA!*%85j_Yl``!4B|lprj+F0J=7+q>zh#xez) z>T(9*SQRrq4W*bp*78*Jx`9z4?XSLhJ|rV`mHl?Vm^0CTilb4@Mn&l3x5V_4Djs>= z?ofykIDz7WNQM6~(5zssASEN){u*SAucS^y$cxzLM!_#953KQd zza-nhR4Unt$xUR#zPLJbTV3lW4fqo3uW)R`&WlV>BDC$$YihyRHia5XP)yo#HsglQ zUa;K%`53k-y>ZBfHZ`z3$e5f7&Cgt;s^RVgC`0Jw>X{WJC8PTwWg3W3VZjYV6Ty$M0e#YRbuSdrC+1G~tt% zZZPHOe`8w*G!(pVELOAmKh=0%A501bJeO`FLVxq~gW~GDxZHkCV>L7!8tuZBTlmff zhn>f-mKiBeA{?KgaA(!#)lT=sJ*$cr; z8m1M_K5%P2W9x7-JNTqD4~Tq4xVc%7fP`+qtG5q1`tx3rtEuV6!GU1h@ctElRHQ)z z-ME`v#KWhLLc@*3@=dz%(!7$hjaE~HbjTPA#2x(*a^l!92@ICMgC7?uFRQKJKTB%4 z%NhPcus?UtA2(Ui)n3y0wbjY>T?dWU!FUU^trLD2w>)nUA}z+kjwpqfGcsLoX4O3vU@_Sq1o3t11qJq zydAMp!ya_mag`b^3jhACIxeasE7HdRUFp!9^ybk5pYbxfQLJ`&=K@kD2#540m0aQx zpMf~}ud4^C3Fk)NWN3H0Lfa7^R_SEa_}@|O2g2*{Jht3i$1eO14-T&wxOTLdfbWOI zwG#r@QVYwoPfNq!E1e#mafRif5qPd&A4L0t%dz+zH!{B7gp0ncoZ0K^vrYf9X!X7u zTBzV7b(~|2Y#(Eq-1TR@bV}wuT+-&Lva|#A#y7scyrs%HBJL+=XWCM#^o-Zhu@)ko zO^fJ(u=#{Pzns-oNRY`mA(e-wF!sINlEeGTMGU`@EB?v4MWYU%kZp?}F30icZR>2= zPvsZJtT#L62Y~`Xa#G?NS+XZCKKcKK4Z9gQ+haA)^#>(;Go^=;vmmNm(2(t*pafYn z-MajhSwa6IQQ?UOgR<5~sk>eaZNrm0MqAyoeGIT>)iqWPHRNd7%TDM zLmW5G;9>j2(Jlmhz0w9bJJU-PcyNI}+JZnh`+Q)a`*v?6%Cgu`zCy^-^0@7!HbM~CYn zpdmF9uC7k>yf=Yf)K$I)`Cc=&$zs$i4l4FAR|=$g@#Q&X&#bK-J$woRX9p7Cm^{5M zQ+*=GdB(YorD(z7*&N{7cj8!C&u5_DFZ=AcMLvV@7}Tgnk~XBn{i3(z9>6UK2s&8< zP(#>(ORYZycX&#J4lWE7;rqJ}@{ed;Lf`2gFvqKeo6X*@dDjqUFswgPwYHuG+Mj@4 zjUZ4wCN6Bclk5JCYx$DWqs7Sln!7FE%H=m{^WSc~qhkcLV~spnMVKGQ1WA#RiS_k` zV&sj5E6C8Pe<;;kkpR{xEFs(g@jt6Q)%C|r4}-W{*4?akGZ!090hmC`mWm|Li8VK| zc|4+AQLQ>piv|}3=ZvNhI%1FnykvTP{SNT+yFHQmVy#s!(JZ__w>(FZfyazzk&oan z@lxLtW>9p5hgtwJ- zd>IDNQ%HOY&y~LVZ_p}f9il-4GzO7)K56lR`|LW7_`;nNB}Q6W)?Q|$mN(4L%+rc< z88^f_UgW=$^V-foF^YN9@!7{!f{CZk{C~au`l^CB-3Djf!;e0FKZnKdV5_9$?#%Jt zgF0*8bPh!KM0s(e2kULxddek~{_Er?@X4DAT zbm9G`oDo1PUP3}4KjS-%iQ1>n>23VY!R)Zx4-^tXj@w>?eW!KSByuz8a<{8=&iKpV zMu?{iIt-Q^ZGh|FKN~|s2iH#`I*H}pWfiA3(=KXk)N+YGh|4r#j$mzq3|p-oVP=fo zOn34wpR_cD5JO$(X=&y9i;AL{>#l^CerwFnsX5lAgOFv0SE1G)k_#9@#4b5k_BpRz z8c{SYvD)VC>aKAi6506onHavw$49a6; z7Zy&f=+RJ`$C2hi4*qZbMjhsN+Hg)cO^PjIiz!Oxnzk$(tFFC%{`&x{_v7Y^E}ol1 z*9Bm}LsEZ*3vx&*n)1IK%R~f+hM<7!a&~0gKb83I}DjlpV z^DeMY7USmFTQWh~z(-M9qLeetB(=gwr9eZT1`J}8zb0I>a9VzU<%ILsuQFAb%*JD( zaWxcnsf3b_t+@6o7a;{9HuC?#P;@u#i#x)(y)~k|!3{F$K{dnBY>qVsVQgtJ$g@a# zZ+8Yx+5coX>LK`8Qu0jKaH&cU_Zgb;kgYm}v127tInY6JlESW193$^ZT10xM``X~H z^ZAG;6?zKFUW}WGp{2O(Ij{Rxt@vL;(Df29wVj_;oe5eo4IH94W}joG&y)O4JsrV6 z$wYVyeiY*YA2<|~nuk*SqgHFC_*^I7dGv4H^Zr5$MxHaC9 zbU`BCg3g6OCdbS0j4T1{+8Wja0*$g zB8|Uvxe+QyOMZR)*-KFv!~^78|Awi)oom>wsXh z(`A3kGOwwmlBdDa?KiN*;P;w=ZG$dBnC_52MrzYy3J`q3pA-+^mYlah7>k3Q_UGm& z3EepQDWK~2bLcZz%Jk3EYiH|v-BxesU zwuJk|%1b9pqe*&Q(F;dK7DnOAnJ^+2lEpvUm(h9~IfgWwTMS> z4u4DAbZkW^ibuv8=~z@KBHP%~Xmz{JxpHPLIC9a!=`2)D1Lg9#GcehY(_Uf6bc8*C z4nR1L^v#QD^%mg3avKp5Nj`_QyF*ruw~fvXChb4>7n{Zn`yBh6bhpxB{{+L--+3E$ zwj0dJ7~MvVFq)Qm2F2xx|o^+qm7VE@e1`s@CUkfbKpu5X)LGz6?8C8 z=n1+Lv(ys2G|^tF-F&}}B>~h{b?c;Q`}2VH8-T9uqFt}$g|?s>mxkv;kpFTE-WOOS zmAiKF|0;o}^dRx$LQ9^WeM2GE4?}&qoyw6MHLK>FHM1%!l=;#>tMqehz}rW;MEiT?B7uF4NZL1OQFKk7PanJ|7r$#b@RttC`xzg3tHafxC&p;_rr;(AsUh+hov;oNrA$JP^ZiGTAZg6i%| zSsCoaj2Gf?2yewQT~%#H;h}tF^ui4|AD}7o^IA64&zgF;4-K#JUVTxsx{osKeOoo} zT;+9eZXq_3bs6baE+#j{HJTyt#2p%N)tWWXG^=*GK~<24R#sLE62VX)hQnK%q$0!Z&Yyiiw`!svcNRM)$+^wd&vAX!kPup9*L|6Sb z@$Z-Z(HfVNkqzvX zP}bB)b`l<5naGF~bZwGhb{$9_@LI!D*D8P>1RfWr;?>OTrh^xmUfFNX{YTzDx4!y6 zecYvucok_&r{WA(4eQNIIUc-62lInKfS1jQ-jVCbyoc6^8}1476ak~Q3axZzMJnNz zs!%+jzUVhDux0>wB#?4)w)g7$aHR4Pov7v|e@Z-1B!<+(R(Q&m3v#CvFs#^Mfc3zn zKMl!LZ4B`P;L^H-fV`3qWsMMGxkTwW5Az&KKM~?9&nK(-Se1YW==mlkK=O=h^M>q7-SAhBId%_LlBf9~9h5Js}{D*^CoIj@)PQU!^Wai@JZ!G4UZpo(Jj`enJu#KAq@{)&BlK@LU{o8s1ev!GBQWTr` z1CL9YXuVD>2iTynVMDd}DO4a1Ge~ z)B-gbqDp;X>c&PmzNkk2F%>BEW|HBvxR+t_OnB2XRez!I)R;U-5G2AD*O}G3+S!d< zeB#2C;^LM|;xWwyvCC=yDQy#ki1^6=a?w(C(q9nsCD^|pmHAYz$8UTzZWq~*@wHty zN=`>s5wmYa={{9nUtQN`J&wgaL*B5AkD>U=YwiN=_7)kSii?xz*npi9^_yj(@zm0Z zW>iS)NMc2qKja7vnU$waUAnbAU-YxOFx9a(j~@qdF8;=2rrfQCIIAhep*`-63Qt*v zE)U54@t4Wt4O=;Lcn&ywYOpN7H(xrrZslF^w|Sg9D<`t9Z#TQS;RzpS9)xbaL!t@9 zT1GsZ3}8h;OONlb>~D%ivFh;#wi7m@NNUgA{5}!WTmpReY#tu0^N69Kl4xy+C4S%q zr#Ej=TRv37Aev%zDbLeC{2Z|nRIYa^W&~Os8d|wB{3-e$U$*7Vw3I}O(tefa9DMBM z97jkp#&2E_RPVtRBPT?m#RL+j9-MwaHG^yiA>=%7KZz5OA;?DgYL_chj z9w`&1>gs?y;v_@1d14|N`vm#^2@n4)m!zGsw!^Q~-N!kqxI_nKs5pYXo;9ElC>XBt zEhdeMrI%uKm=hG`EgD8O4m3_9Q&bG%rZ*wBxRpxgi(zvaThjz4byD=$t*)gvbH!PB ztZPSq4OJg#`#6!Ozfs73!o|#`G0>wEkZD<`$qFcaT8KG^1;HMD^z7e{&}Ya?`9Uk4 z^0>q4l4ATFPzzAcD{i*|O1*OiLnYTOzv9*V7G8q#+*yQAI$;?6eN&>AnW3oEfl$Ep z`J6!fixTLBiju(WtIHeZ2t&w*K{4Wy;@*N4jG?<%)mP4rUaegpAv2A+iY9qFw1K4g z$XkmteRJ-&f5jzf)|7P?*AtDnGqRKT1%-JQJ_s|HF@j+9ZdI4`ih|$NI(JBr^p(&Y zIcZ~kEP&o@rS|_gy2_}ynkAS7f?IHxV8PuzIE%ZxdvGUsa0~7boWZ%DB7r?|9oRt-RD{g4b!py)H>B(#u94{r@Sh30Xcw+n^QJ9XD zQ8JCX{|6iZLP{UddkTc&!HVLm*4rsgy9s7_h$@S6abmJ zFcWhGiTOKW`sCkc{uEAS1qJ?2*Bh2FNe5IrodlgRSN(yA#u`-TnKYfs7bTp)Y`9Yj zc1nY#(>`q2$RC%(iMP(qb)6>r52TpOa{HD2mzci%X7Oa-gHz9Cg%0QAgr~8&lJ`$;3>9(~R9;WSY)sH|o1= zU)~H!eg+I%X8iM$%I4D}PACweG@hbNjuCaD$RE5pnHfv}42l-N)FELq%$Ai25dN0- z8*p<}(S^8i&zynyv*Dn7YcqTDzfC%ei=V6Jc`lPA4F-j#i)ho^aD)%I^_ZFUdx{xg z^G?CoLfu!W_K=_|BCmPzRv1I)qL*l%+0)~|pZG?(8QU`9YqT-ZMVDCx`TYY2eDrr| ziCp~NIPksqR~Q;o;90mVmX6YM0P;^eUtmtlVycE@+%0wat^mqRO%AKJjg31M|7TGu zK%(!^cipa~`!Sre(ZcqEm2C!ma8rTR3#jla(e%|)Q{{ds8G(qRy2GB>Arbcujt{e9 zoX|+XQU&Pe@|#pu%zuU7=Y5XJ9e`Tp9YtqHBBzyp({})b=^q|FWD@{OY2ub6?0YKJ zJog$CAzDF$v#aKc1r7J}HC;V9I3?GP&C|CEupPj;8QOKfR?MI%1pykzs7V(q8jb4< zs+9ojDLEo3-ZKzywhut{fL!n5g8Istdd=|plCHR>#C>Q(F#jwG0H8CLw~Y3!9tvgn zc9sI#v99i@6{d&2ox;qxZrrw+;+XuL3lx7!p*{mTq*_7Nkg_Jbt3Ba)#Z@T1p!jJ! zaRQYpM%L2>3eYhJLv+Kc z+sXpH?RRegAiL@NaAU}saLXIFBXV1&`RvpTslhz$6U@W=j~>QUV!qpIp}Kx28x zybg{y!&xU<_dH=R$l8=QW?grmMZ=3bBb;O0Va#)@dxC2N_vu(+lY}5cwCnecnDt|^>+bWCN&kx`h#$)+!+yrpdXolQ%4qsY2DOW5NKIdPH zosW&yU{9-_%dZ^d=)ysN2WH18DYL;kT$orqk&!qARW|Tikz<+SrVO9R`v0#+?Dt@= z@SIgzTFUL^av?jUnHU$3!cL)<4F_*u-r|()H&F~Fji0#5nd>$Xf}lkrT0;maG9ilW z#za?T`u?#$S?S30E&upgiqsMCajWIen8JMd@b6j(ne>+j+K;nRP2NeNrU0t$R(m9* z&0cXFddV8AGwNS%(Xrm5c_i6N4S%!Wgv`UIoGgKOMO67b5WuBLC!zR((u5 zrBcujaKnJ2s<-c3RBvMfiP8|E!u&hvLk+s}_?8V0Oi;bEcBqm+2CX&$z&+pmMz#pW zeF2546*ni#o#lS_&=n<=g=7_=jpC7O8&=wDmmXD;Cw8wDhfz?62zDcVBi|WBfaDB{J3p4)-5QSW+ zPf8}j|NGkYjV!qCm#rTy9$;cyq)g4+HRakuSk-Y48H~Oo;NraGhy6M8JCEb@2Lz7H zDfHPJ(iyRDJj5=xm5m%p3{`Bum4VrnAXX&69D%|?GnyjT!mZJUMLKfK1ER#YIW_p% zR;#Ro*3be46>uf~IWv%tnT>{y+) znH{t+(GYAJjS64>#Sxpr$NPCty({JJv5~ihVQ{3{BQ~YO< z=UPyuFzdnf>S(>EESc+lHO^j!Ixsj9rB6l*%%}I5?C2${#Ro>@>$ZCUYn3_f z^j)>(EGNB;GV4*Z)zYw+ZTz=*pA$Vm-PO2_Xiy@gWPJbox5tLh%|7rBfMEkR0m&|@ z=W6VLx{P^6V~IA#V8AS^%q7TTKR6#RO)nGhnRy=a_p!02)l^cto&f>UvFRCHs;YY% z%{86POM2!s@8)AEc)hRrhr3t74)T?xh!P*hej2IeLmO**ZCBv+EXx|lXK`GKW&gu# zH4^nzQDGQkr60pjmq)e+X7c25YhivWlWz#oco%EXl!_Ql*&^ssp({IF#BGw$Gu%~w zokSq~I8q;{?8Hu*%pFza;$jGO$;Ts9){+aebdPx?ObpaZ17RV#KcP8bv_kH0=NgHx zUjP~V#okc~)In-Am!9ie!dHH$FH4Q~zzFjPV|Qjvl*JXT*a(q?Jmo27^fTP#Tw`dc zj|RHQi*a!MLA}0O1;T9E2nuzdPSMwyVSGCcbX$MZeM#-jb#aThxX#Tj%R2@w2Y&mEb45Q6xmScEVw|BYl z(rD3}*9_ZNUVl?jVyG4ShGB%;%bcFKj(CeA&URW{W*siZN~{3HmODU!ny!%7r#4TXELoqbubE*RQ3eQt%#mco$}IWh9w$=k^}oPc^a;7h+=X94J1VmD^OWVt8* zVV2PK#)O(*ZmN9_M!V>F8QIn4lcpM z@q$@<;TK92>2yS;k?b+^>n~j~4St$klr30w{|O4FBjzHsd7}#QvA9nPs4UX0Jj`)5 zAv5WF_PH;>kK0p+f5R041J=2wZvG0Z1D!T|sT7Mziyxat2HuczGfI(#1!do zV?LMMqnQo>j3fZ#QH2%spKzK1!x((}3Wyrp@@ZIosjV@cWRK6=D#}0n$h-G=5ue6> zLGRTyH$mMk+!;G5Z&`h%lID7T&R^cEsbV>QN$cPlFhkpsGVdiD+4Zb`O&MNS5*suqedA5x_rC5$N={8AZI*1ZJitr*Vr4E~33v7&<1_PCVa0oEtj?B^U?ZAgVP~mzn z5c&=qW*!*Y{PJBdXsGJpfjZ?h zE*Vum?~R$$E4l3J5~>3xI(9;2W3e{}J>{n+BfEZ$N{zCu(m+^YJl%i(_wP5-39qsc z{;u;pa%CGOEiG*gS2gzr2`FSF7`7nd9y0C^BV-Ozt7MC|BOL;Dt936p=j)7%Zi{Xy zAg7-nVY`JnV8E`fDGOg47HC=PQ8XUDmLbgR(X#cCuL)c~u;5nI($4H&9sP3j_uKJ| zC{d|+PangmDB;M2OP>6!S4~oxAe=#6_u7h#V6H(b%x{JoJ z3u|X|5Hv{5F$fpiNftY@akzA4r>*%bF`D+bFj<||2(%-cNMdA_-k^T+mTEwwdf&3{ z>bDr`tjCcx>y*)a1)`XVj4V(4TjXBD$XKc0?6~`Q6$Fa{;e7Z_`8ID*&8c{_OG5i45B=7M;i) z`2PFXEVVy$@lBiIN(^J%w(^I3zZy;A^M7#fA|fPY`q#oKr8HMs+OogjaDI-GGP!&0 z9Y^xwnfOMmAgXxZ5)gaHq^s3oH&4eiIOAG$KRqv=*dm#{#^^3OwZD1P9+L34w6t9* zY365(E*X0Gq0>Xr6(u~piv1d460FQt=`p4C-IS4h18;`nbiy7hrVA25ay8c>A%~sE zkE^vyM+&vl5Ck2!#Iyp4rT5NCEqh3@3;r!P#JiW!_3|rW*w34DF`beZ#$tRHJB`*B zACFHSHg|~Ift5 zwl&xH%=B1nbRN6e(}*{}oJosbpvFk1NGY&*oyJ@8Dc z9}pMFTL!5zXxu&0;yrK21zm`knNIW0Xlo%H;y!(>vVTKEf9yu&)OOIv@>@b=;;V9L zB+!T)ehnD{2A#T@UHZ1tEKt_}jfICVcRgho06y>q&sVsEv#X}V`hdXn#QG}Q!@K*wrU!y9^9d(=Yw+dRn9*3w@YH(zF67EGc|L-^MG zA~D49s03usz)3;xy`>r`06Y68gqmq+7PW7&Y+_7&RC*KkywV<>(Tw=eI5$rM8DS7Z zjQ^cXC&)2ziQCy31O;UR%t6{aFcm~@iHZ?hLNOlwRB>#Exe+FnH_tM~ z=)up(RphXvBJ5=5kIW9h#V!{3e$?N;eQkPCG!UXwYZHJ3<&BbHBDxTL%)tm>l z(xgSa^r4EI6(0KJs%!ZDI=0SE&w!vEk9cF{f+{lY)Fx=sJ)3)_er!>_t0#flU|Oss zq?LIj;ykU()A=2U9ZYt$AUFN$uY?5F~|;XsRN8z2jY z{eV!Tzg1OTzHiSKK|xoLB9M?;Sm^r~ck531*b%So`k+(i;bF-orIve{Kt+*4d%OwR zPedZq^$n&op>4|`b7jPvV}?e9QJavoIb&{uA5BC~i?9?;?IEA%!;S@HHrZ`FA)1my zE-ku7Mn;s_Aa*V;FZ{ss&|JDB#QO%SD9CSOKpn}G(u=P#Q&*}=ta0uydGUcoLgZ;n z>T&$0;WN)YBzN9y5ID~h7Z>(Ui%$V}cbYYtw)XZ&(46eJ5S!A(btgX_bst&tol;bf zlh*`#i@KJOkR-9*V1*otv2)%oh_k~V?`@Tdj#taFQCQz2DsUDF!HcP3ceY;Yc__k-nA83RqjeaX!V#mAivCZ% zrK*LZYKE3`-{;09yj83ADGOI54h9YSjqEHU{gP=KbEqt2h=+GO_Q^_#`@HP*eQ~&$ z@!_NaASzZcDdbOh$@!Twwrz?cC#&o2bObf5Ao0prnay@CV7Jp%S!rh9O{#OKt6*St z;{$Fe3p@Ll@P0s-Fl}Q4c|V}Fi0-N)-Pg zZy$PZBtHgacFE(~v7o`x?R-fHouv`ySZB}X9iUWv7|>dYqvpzy_IkMhjy+!s|D!Ck zJhnzh%;xn)O)_PyrJ5*!<(EScL*U!gar?yp?9oKI4bU@wKOEEWxjV1PlzehW8^SN>MS6mp32O$F8x%_;spwp@FkNaowql;RREjg52bjS8bx< z1*_K1WT{L7qPLOdx{LMp7WCLvA$aIo%yZ_H{B~&&YI9=`!8|OJaAaKy&R*1gX#)XFt;7a=)O=UC-d=%_OthSfo_RQRA34fmTi!oTi9MoMGDn?Wk8{ zUBY-9-O!31TlN&cId)|QQQ_gJJG>v}8EnKvt?2B9t>$ha};{!(>lveVNm$;+GAXLU~3_Sx0WLqX{?f^`Q} z1h4=f0vp-;*^RA|Tdt(Bki?Omr>FA7#KiJ)#OadxFKINgta1|-Soagyfud6pmd%P* zBf5;~@+k(qYwt)=^dVR~dh7z{EI_#WaVRZ$)vM2K(B^4K1MdSPZ&zTGPsy&QuRAAg_X0i1&fStX&bM?RQ z-UJlG|GjN&Fp_3Q`Z2q^G4raoSn&HkFU6C;2>yGGppjHjgiRG1I49pvN=|qqUOrs5 z)qw@)?qL^%IL>(N5QyHlnsvc0XJQcna$r18rUuNG2YU6xQ00}X2tLI@i0Jlm>95R@ zvlW7UjO8ZFakhEY=&_q=8Epc5e2wkxXyJ4EjV{rpS_@d5j*=TFDsxVs8;GVDI=_?~ zdD)#!tm*2PZgb7*@jkO&IY;@#rVu}*5)Pco3hWEp9|eQgW>@sBc#hZ{(eW!#iWX?f zcpDnsY|IBwglR1~8(e(tjyz6#p25T-fp2nej5)Bs2G9W|=IyTc#Z46C=*116Et{&W z_l<}5+_p9-%`u6MG@hHpnkFiYn*%|R_6WVjg7ScD2bas`R`&b5J~K7y+?0i;4yBSg z@sW+qWy3s(QA>k^BMoZp?tR9sXGqXh1PIb9CZG_hlM&~M?1Q+f4w3tGXNMYRi-+@d zzE2->DvxD&tFUC7qj-JbfA9W&{zMWcUrt`i``Iqq^mnmCVMCBljfc0}{SaVoomgif zL_(bH?u4F7!8hdfu9U%s6FxX%TcR}c!?J8$PR6yGAl=4@)$vzIMKsqARo(LG?5uPA zud>CJt>->qrZK~;V-4!_mMTL#4@sVjoDViWIL4F!Fic98XByg7hgB8{H@2p!OILC( zBDC}Ixj$WKIQ=X#kz+kRU z_XFGWxA@nZYkT^2vK+o6Hl*wFtH-nDx&GR8PZ=5#8#{)hdEXMz0Q{=LL{rGIpEb+PK- z^YwmRK>=NOF!@Ojf6v$H$SCTk6IUhC+2zX)50$e<<&q#3$G8>CmIs2$CAazk=BOvB zy~w=0sOUCuDoE6TOmMv5BESjLf%WDV-V=v?ap#+?6|19=t>^imIZ_ldLsfNPW}MFVfh#r6Mlm6snbX| zn;kr0v~g-D>%7}JpdwqTDL93Xb#K{=zVHXdb@c9G+7RGR`0+#C1}0?bO-_KzN7^eb z6;j_hkVKivyefyylM#S84v09jvR>EElMqBNO*x5+vTS#yK-~!At;G=G*#%|&RuNyD z?xm<=5dbZn2ouXG&6_fE)mL9OSxmfMkoY)y8t0cR^rGcPWEYkQ{JtJX%Qop~zS2h% zl=L&=7boIJWD-`1&eTiDG+*SKc4YPZ;lrOPbIj4yd-4Ob{tGq}f04Ci!dg{Jd){QO zF8MB|0eUdbHePw3M=A_nqoNs)_`6dYmN8vXNtq}jBFRRlohzwZicS=^`c9$H!@V0J ze{)-Br(5Gy^Lpjm%geu$s`sm$_v0oVoeUSz90djgy|%hI?MmH%sr<^l`Svm!;@B5J z68U&kLHkxw91)M%npfm3!(SjII52}K!;-$> zM4;(TE}a09Nchml%$f$wP&vu&brX;XxqPpiH?n28_N^W|VWv+iPW{>yzE+?eQO4WS z`n)loAp}@{-I$o9cRtaY$g_0=H_}t|#nY++cs>uQhsNXMiH$|=K=-eN#=}4TFI*0t zp8GPi783~!#ZxTALh5#w1eu8sJ=@3vQ!J)O!G~vDW#v8ix*;QS|E!~KSJ$eap@|vd z$Fm8D)&)hG_oomd^i>1iUfc!#y@oF=P?eLj%4ul4Jaj(5`rk?#bY(ynrTB9^^;8uL z>W)^bolE1)(DhA+i?L8jCoG;?SzBMeknmO9U5^38eqM-M8-g~Ank-Iz@mhXGeER@I zrHoN1doP)?N+;pZt8&F0lw!EWjSalOzolTgF{~T99~^JD0bkvR5A90n&NvzvU=IR5 z9tEhE_bt(rKmz>A`>M55_uwK~GC0H;%T9nZrURU@MW)VXewOXqklZAvKaCLt9Np3$ zc?$_T$Zv?DiT8AjEx6I0{-+CWpigobKX=uK48P*IEMG^dY6gHDvF$~iW?{eO(qq&; zvlB>aV@get+$ne^W2?@2Kb1dM<`(f`r#6L{KYqTV!+s#3l6 zJX8iXk(*rkpqG>*X7jo>cXb_>c41A3JAh*=OmjK|=|O_><~A5e*@L-Fh4R<79^5jE z4Z1b#vX_{Q%c~1p7uIf{+q+k^*@-t_ab)&+m{DCy+_=R(IijsaqW-emG`32ssk%D3 z-mMdGO~gb|)N&&H75l+@`sX)nCnbhy9|aee4TY!;7=mv56T`6;hR@{`cQJKT@!LN| zs`OYfKj(d!b_pSa-lC^1JZ|eQ1q0&qZqgF8faCGE#*a$M7S$T9YKp)4oSgwvTq{@7 zn7n@02Rg_41`c>iOOIdsZZf~3Rz#5AiZ;DrcWaVKn1^fj5HrBJ z=;gqNZ2}F2YHk_IGM@lb@tvfR@hWGmt>JjzVtFE#CX)=clas(Ux4XuA_D^gh1I}ZM zbgF|lMnTh88Qa@Y2@E0Ai;MG1kH7B3_4LjtLfOv`~ zd71S&pIk)kGxyU*8vP|yrfwR`%H@kL?qxQl!s-D8**<0}4Go9Ce<_?>?3&p|f^3BJ zCTrT(proe^#}a5W_5+D7`PsFIM*6OqKR%qcmRG|$XYhqa2o61KNyt`pb!mkfVPcvY z+MgfdFzV){m&q4nw~b%eN9Pg9d@cWTBdnw243NBrb{{ut{kq&YJKrlS8)sqyB-p7TBzP{_ILXf=?Q2Mp=48asA+6nq4>hJQ zDDrn7n7fHcX37V2c`ykuP4LdF_@D$BT#>`vpbzirztX-<_NWcuwwd!R0qBn#S$Yiziga;i-z5Q5SN2L%pWmUTNI8SETYiyS(5zqSlOWYrG{XDViX5 z9kPtL zJSdbUC&(=CO+{?=ANzLCtYw*b5r5CCogw#g)5u1;T^;yc`pGXec7OEAe5CkG z?oQa69fa*(x>{CE!{8cVQ}GP@Y|MmS7gAgnA5QuU^6S3kCC5!3E<0kBjk4taydG{# zc=~KRtjG?|{jCnuc|BSO%#9lF%AG-^6DV>fHME_0 z@H2?R<**vc^hqbG)|y*6b5CfuZahWvx-s1J1|vt9k}G3|EI2(Ju^2=+K}OC6IkgmR zdc(DO$Ac-6{f8;W1C9wy3Q{K#WrtYATJrkI{Ns638xut;Z9%@^+9B|_(Irh9_*YY3 z0qY$)E8IP7^1_p%{qnmuva!#(t7&?Sv{n%fe1Glh4O^EZB3N5$uUE@8CS#&(6DiPe zO^qVvzm`Qb%3Gg;`O=BFDRBb}F3WC;RXA-!9+8Xc763S+QlX;a^?IhzxuD>w%-yIG z6}r>yXlg{S?UVXuY?)scb;%#Lwa9 z!NK65goS|+ASXnCe{Mh0hJn{nC#XHguqwJSJL`FS6p)-ug%Qg$dW~70VV!e&?XBzT z_jEPzeZ97Ac@fv;C4kBxh^w+ZTrzrXB%z5%)jzlRc(2mXArkU~E`nE3@TD%N<7UI# zKZKq6Js`h0(PY9T3+uv!uPhH%CHL?03v zWe2Xs7RG?M&mFWXqd%gv^E%?!A^l*T9=HF<5nFcJ)!}v)86U4ZAEtfs_FhHAd)tws z|9l8&I68YjJg=!w2v{A}1q)eN{(MqOK9B7xv9zJ!Y?9Ls#pZ_my=SmrGJC=PrGMnV z<;5Ukd0OQb+KXJY*f`~tZ~J6?bq|5e(_)8SdGh;$!G%JEkYk<>3x(msik{voR!lIi zoK0FA+Y?}matpw#>aY#2j47urhcQXmA=;Ld1p1Q$KeMxA$R+PNI5<*0E~VAYwKnIV zgYpP0d1IGDO3V>mTQ3^2XN&n_?&e*6+b)tsZxg)wvE%?)o^SR0+u8X~)G1eAAD;t& z@BOmwE`%61o(W$OoUWf~#6xmBClkU|7hQY%#r-`laa}EwL5;)C%k?hnzZ4|Kz7PFg zBh@1*$hSVGT2c1M=lqU3v5(%VIt#gY_=@+=dv~e)?|5qmqU4uoEl2Z=Y#2j zT6!q5u73l*FUck>poLGVK&>Xjd%-}RXf^+7|!=zX1-o@!71TQ_2ob<;T zJ0m|H;v>5!?MW*`pkk+WF&xU2BpVb3DVW8M&0^Bt!9!C~AO0skxZ(Fwcl`jZ9xx0= zC()`wHMb0cr!`CJe_2b|LgiUj1FJf6626C(YjiF7u?0{fiZL;BDu7y&Ch*q-c~K5;F;U%|SOe{{HXnYSEhMqIs1E2QvrVQ0wap6R@A~pV4b5jqUe%di~mc z>$8j(Ij9md-YAm2JOp98$HBjuYeHe~Z3XhmXoU!am88CtLxKjuB=pkf&Y=beh8#ED zvU!enUS2LgYOE*WpxZY(^z>>eKI%8U7&=^#U`7J<)7X^uqZ`R-b{L%4CUfGoov|AD;o`Dz1XA1;Dh0fjRzuckAwx++X4Qc|dKQ z^l(It7bXV*f~%^`jtBH97Bp3t$~Kw$HBR-ikKABLv5uzkv*xUvr*JFO@Q2aw>uE{6L+> z7N-L{xl}8B)VDST58DlOrnn&^{MjyIJ~B$J&dGs3>;3ZH&PK>rzscL(;t=uWzQxR9 z!}OD(Woveq4(;HX8wqdyP>V_+W6Icm>&xa>E!m-Kc=4t!qI|p+Ll2Z;BC=@){!_z7yjaP#I2$Y8WfV=B1gvrhV2zcD+2$T%l5elf{Y(e5yeGO zyq=!j__!GosXzn;*;p+_T35?J8p*W8lHH4(yWQ$Dj0-tZl-UjrF8I6eS$>&*M*6d{ zhsO(MsOKNdr*FqjhrcitYg#FLV205Gf2wpHj=G18{7`Sy$FKy!92^d%q}+Uk_i6tm z6YSzw4z7&L!^?J-Vw+UBpA&_9-6y7(dA6(6081wnotF9tW{+sT->qpI z{ClC6_W-9xGabq3GaQA&gUFEdxK!7tkgFDNM(HL6Z~o?;^FMf-Jo^Hex%YMml$CDuZF$X zCJMy}mfkQm59KH{E;a1bBm{0I4!*YWMB&+r9t54vaBwynAd+Ol!YdK1N?e%U!OIjA zI|;6`QKM1o@|B*@I)Q|V#;lN#i9oe0o|_tmf8>eyqP$OHP0srC8hv~mjO^AmmazAu zrKJ0IKt+!70ET?mYCaf8M0!6m{n=jwB&9B`@?FA9&*bvAlAoThXYyO49W5w{6u zH46}Q2_))7;h&f|sLbouph;V`eBE+uyO``uFDx$JeLBoJK0f|%8<)5NGCi#b3TLZfb!bRy4cCryJ&?=7}DUCHd3 zRZ2K;@+Yvyg*SfuBs8h+3-HF%Wl9^?8}&oJEw~!4Gvl2QKH?$uL>}w~2xmepkkqL! zUGf}FyuA1>d_NQ=0@i(3QO3QIxrH|amfB1!|E!yoB9G@+PYX^x#~LdOlOWLzJ!y06 zZ{@`bBz!)P_E@v)ZSI6Zez||&33(!d=;>=)XtP4*#aQl-kk7pIM_0{g|%1M=e9`uTJb$gKsU6^o`pU19*yL1%=wHnYo2>7`XFW3k}in zR)}gTOA|shA9-+?*lO<7er2$qk)j<>rzGPy-; zWjY;@I%3Ieg?%GjAo|#z#!dqLh#>45`?u-(bXKrT>aZ#;DKvOuxyJ!jX=X7s-FW_% zysB6k2nsgI*fBWtTOoaD^#6ooxJG9tTO8A_#u<{9+wV-TulxIr>CMmbr!;vxa3g6r zct}b_pkcPrqj-!2r;r&YY(0*Im%ncxKp@{u6UU|g#?fGgVq@RJ3O?4jJl|{^fdoTg z)7^4x`IHY?T9OS#MUQ}VCM(TMUL zi3NjxP}@#nm4T+~?@U=u21ZjXdEchbRv&PJEf7L7*w_jpBa`XBINWp8!|Jrx@@&4# z&{k2fww8HzY)nin3`(%+dQlpLTs9ncD z9yVPr@B{P!sLcuwVgJ6tTZ%t}C^aJkISee=f{z{~PVExs9yz&nafC}w?>B0gDI3Cx zVBMNMY?`3%fKs8wE4IbI7Md!WAdKT8Ts&{2DK`X{*gk41o-@FVq&8cT4x_=&kSUwo zI(QmdlP&8f$WK^T67(%sY<55spRF@8B3{*q;VJkN8Rwx}8$WRqJ*Zoo)5-I0LD$-O zpPkN#r;Zml?bpF>0tE*b*W6+Tk7s%Az{sBwhqArkJS%J7iwj#@-iGpRkDRJD>1*2x z`Z)TQQ=fn=aAzaT@)<35WD4UtK1mMvoUWB@;~vIcbgO(@he&PQ>Zg7UzFnjfCRfcarP|Js2?fH`Rn6hR+=Z3`65vfQ@Yaf*UKZ-uBr*2p*b;VjAMw#F2_ zig!S5jV9ef5Qr@4#-6i=bz}uVb2BEi{xXbO_ujf92Gv#IbHmuC!phrWohFI*UH{kR zXaZ3+f2AD8snzTG^pdZr>;~izv{kZ3%fO_M2jjfyW&gFq%wpwkzBdqkOl%6{-r$cH zIiO2|Et=IRE`l_Ecq!m>SN3EJOQuf@`dDtVgo#t%%c#WFhVP;m`)3v?SE;~bP$tK> zX?eiykYZ8;0Ks%u&GS_ePd-_Az5pGN_^dt~=VQ$2gg_W;(|n_*7IiPh!^{ZwL- zHGbC=QX*DG29j!?_AY5gVLGx9hn0PC)+u8fR-B%=mrjew~>lTZP zfNX9?v%41j}*X|si8KOVw_HNie>uqe>iAE140vyjy zFIUnG4$tSjyu9<f-;O=M`qroWIj%Ds16;VP ztK~aa#7U*aJD`Ijnd1+%w&#}2H01V^T=eg~@d-}dFU|paZe|6VKwL~Ngp_YYJ?T)y6RY_7EdnhXCm>@ zvqN+FZs(1emu>Bhjn7va0vMSgDbl7q!oUN%GQk#yn_X3K_ zf{3O}U2y{y$?$N8Al8Cdj+mp?;yQJqnz$C8u;7COAe>#*^@zMwbY~)lrQ}cZ`XeL5 zcn~J3OHaXtfl>N6G3%{*xzZ}7h)YiCt{z*am24)bf0{D7TTFOjm?@~iQ6rp68-R!y z!@y4GN0tB=xwu75Qx*sA@jp_;&7ni8_fq)lj@0aRfMF9?LpV*(OTDbf80MTQV;4H= znKkOUt@Bd{s*tUzskM)fxvzlR)AdmtPQm@lToB!-dCu z(a{jgGPg@_Sa%R3fEGx1+qU&^`L&jxKO<@vf)k&ed0P#W-JLOM0!LzZmhYtPl)tC3@!UVR~k^wVJ7KYA_OC3Jvee zyyO-S7Z*3XJZDdCRv7_!nIE`2Y-|$e18jc*14to(#IZVT^#nFSCRbMF_l!+mSzd%>L)rqT674vSUaMfZ`KPycO zDXd^#`qS99UHJoN7%`(i?X~dDV=n9pu$#8oU0m5142UHc?`rn<^Fq#QDO+U%t6@jM zT53k#xmrF=YV*`kP;PX8%TK&S)NCl=p(#qj3jSNlM{aXhBu*7!A`?&aM8uMQwzSF6 z&}(BG`;g<$YD{I_6IlWPNKbP%SIbkiIJ=~Ic;op|ZLzWf8EMG2zDkM9Wn2+-%8{Oq z4>YK1O~OWOHZQ~^!e(b9#-dnc*1uIyC5R%n6j~l61k-1Lc*n^dQLJ;AhL3mzA+ zWld@0=2Yj~B{H8vEK8=tPfx3oG5T*eR|wXPA46>UJg=qPn|v*(7o2OP1dEQfRaE+F zOkioh00vn#svEPI1CFc$t*TM$dRx9gBg(<|6yWbWJ+Nqj_5!Hl^F9Lffxf4R5*v#A zKVTFwH0xRZcy7G$D6NNtntijj24u_%Ra&dvC^Dx!7%Ysm9WNOTRgBnDidvZ&eGqYW zSxbu%fXD+doS|Stdjf|iQoc_Uc>z@X)lBPnXlU!khWspOrJ_0Ui^d?cgMlkc!UWS$ zz3!K#j<$=J&41_N7-1A6J)7=8>kc-oIy)T;MJWlf;aFZ&_*VzQj~l2 z+Pu2n?+#y{X*I`}_npo*yULwzT6fA#(52SbGkU#60z#8 zP?xkeoy*>&GB!WHJkZi&A`lChu&a z5AkZ1zhO^x*X;WyF}wWqE4`%+YlmRdW1H!Hr!5KexkG;vATO+Sb#;9o!tuYeywZ z=`XoIS2wr7H?92Yu*Nk&7vM0PuhT|@NpTKSTm!N!e7G|T4jA?j&-s21HzHS5l+vmc zW#%+iqb&3Q^k*)7FZlegaQx?1SGD3F%}QHZ$S@H;4Oq6~$`Onftr~jZ2W=$tj@gCy*yji3BV(8yqV{q=n4+3J7sQkmjoAHqM+5TV5 zt4#~)d!z>pKRVjZsLSn|KOC1@5SKMYl@^#y|;x=-DxIuZZ=+;e# z2Stkusx>@{Xq;4%KYoQ^QdR8zr5RniuKhMm!T)zZ!P|=$=)P8gSJT zf0&*Ul@!DgBrw%61}siq2N^uhwtUn(5x-8#<(Ls!LEwYlHhOMBnF z3cn9A;f~(jF1Z+$qQfq-hA2b9^Cd#*yQ?VP3wlNA?gf~`{fhZQg_wwIHu75zvzPPh zXrF4N5iXEf$B##h2lQTeA@!1!?TNZ6d%;qZkC*(o zfmR3Gmnzi)rvI4#n%|`}mgL?A9dtU1X-rqWQAc(WZW}lWFK*5(drrOfAONpt?BG*u z^GA+l+Hf*#iHMCsgAl(BSEn+Lozg0l&)%cHOtlm0c?v3RH*82$3;wGBOtSKrfRhZLu2Sv<(+ zDqxlBcntXKNle+6py5-7EjDXcs_&sG|Gl9F7@C$WUDn@w|uHrFxF z_GS0}hU^nGk(1lKA9Vfwr=@e+6Z*UXOF}=BQp+LQomzII+m< zr0P?W$MxwC$>Rf_o?JG9d)In3%nUz4+NOm|LVaqu8(L{;`{4rBBFb9P!YhEI-ira= z|2jJ|5S7rti*{N#%zyz(tYUMRkE?3fSkz`}v#p2ze%%Vy!B9%PICPwmVgU<7Zc^Sd z{N_&-sla5`shzI27qAa6!uME#Qh+^z@O)AbF&K|sFCx9I#?Jb^wASq6o-4}((So__DulE z7uT7Xux&g3c{Q=mqIg<4nMEG7G`!L_-APOWsf2rudpu`V16y+z$rr7fH%j|A6$vGn zY=%dKd6P%NeP%(-60u81bsl&WEK!EuAl<3YVD&IETa&YZWQ!G)hko9r!2(sFtqwVw ze*2308o@8~TQ(-**5iL9U1d~NUALw|KtQ@fQd&VkI;5nP?(T*|cS=b~NQZQHcQ;6P zcXz{G?{|MP#u=P__TFo)IiLB&&%RhNDIXqFDZK6xlOX9xmSLC;!>Uv6rJY0x5w@q4 zB-a;lZs5g$N|@F|>g3oB_SgSvjK#krYcGZ3@mcG2Z#30VnUNi{br`l~+zf)hC@7?m ztE_^!vs;4J?bOUHmt8k{x-w%?4f>$NLy;INCX>$DF8?95ijeB}yb(hG?b_E+g@rVj zppdPcK-4kGKGp5RI~X6sU1=jF2;DL`=#3HlN=k<$%yibJ9gS!$9OPG<^KGNCG@XF( zTgY==r;5ZHu0?nu;JxqH)U5Z27Qt4KY0ZLggIb%MbHR2p&X z1$iza@C$L_Mc@sW`Lg#tV4%E_Rv&&(0fx(x_a5h+fytYjlS*E3CzmyxcMF80fi`I3 zv~_2P`bV-Da6&GPqFJZ5o?^&-JZ#-x8_0#qeR9phqWg2s?Z7W2Nk7`RubaYacI$ZZ zrB~@cMRP0R@5wtnJ)N)08{56ynST0iT!T5Uh@+|A@CBcM?YaE$?~WR$QCP;7U6UXG zz%dNAC;>^TwlzujDUy<-RNV@U@LHkzVu!|as^E2 z7<0l65UQdoxr_4_p!Lpw;h2Y7SUT$uj$VIy}tG-{Qu!v5x)9}82%9A)dQ2*SG6-b5n4nMR(5)YY`2KHdK0AC z6TPsnEJ!sYK2%PYTNGUswNubSE}<}k0=Cyy*5T!ey$hV?qo-;{xH}09&E-Q-heW_gQ==`lIe#uN1R; z2H6w25Rx{+TZE1Vcxy+ZwgurD(>;{0qo2$n4S>Y3i9^ zjMHY#Vkmf5gIFpmmG+{EAL~=Sm1gQR3IzQtoe0rYM`G+{JF#XJH$n%h^_KDh$0y`V z9OMHy1{=xQFw!`3Vtp5sX*b6 z(Gu!&G}iv1>gTeCmfQ{UFPVj1?W)7YFZLJy8H;T(T4QuL9u7nNkw+8PPctkCfE>I! zy;XXtarVuVI{aiuII7>AH_z&s@56}kg-x&*?UC-%Zy3uQs(M`>nSj=wkEc7Qts92kLp2T;bS1V>*6N-#=&_lw+Uq9> ze6X0TP-YT}H-B1~$1YQ8f7Kq*#;~22$IJq-^4$5eBq?5bBVmIZMAyrUL5Y9?T4dBhWRM&TvWWLS7T4K=^x#rOiJtI3;Bk3>k{h?1l8yS$V5-1n}Qblj5t z6^Pmh4l<83g+|3p?YD5`@;^d12k${_>?z#SHZV|T|0o+paPO4^mSrD2iS6CWt86Fd z8Wp?pjoEUHg-X}2v_@I1$PezzzG?9fu*mOA#0V@f^R5CiZeKfrZ63GkTjvK>ccjZwss32aq@7&N3y;4B7HR60>LkI zLKtbL^4R;3X0L_U@LlJL^_Ag-S2T6G@zXXP3sGN)`_cGHz4hKZ23@!?_%l*HagyY; z>Iv3=TT1MX5`L5m&=MFvp0;pqC1}Q%nnE_V;zeaG3QkSH4;F}*%e<~L1LG*ti!(Dz zo0_r}86CMeKF3v?rOY7jW);mtL^-WK4|R5a<03A332|8@f>|Ds`nF~4;=5-#f~g57VT{W2G%-CQq&3%Q zPw!y|892_zCjKSy{YTtBkH%zt)gNiLQii4h&*SdR&DK#G9e0re&i3gdXRo_1 zom2sDrnpLz!@n#ItsH9^?>~0tO5X7h@yT>l$3$#x!j}}+O31dBqWH;wF5HXUu*AHY zJm+=S$X`^%RJBiB5xKI|`w^=PBjN#NAw=@>e@E>kPF=OcRKYEQ(fFl?vgr3J>BKoSC(o{|wAsL2Rt7xIG`b0alYkhSlTP zxaZ?BP5*LYY-8~}-OfCN#aqmI*VNza)=%KAi`$nUv0Pyn!PmBKk$L zxH>rr{WtVaxc;jAar-P#?=9%Rr)|($sYP&ZA?r7+{39(}CSBA;i*#gqUOO#*8Nw4? zF>TqfxRnx*XdYE#)L+03%v^KR} zi97xf%KAY}k9gSs5rc1avxk3>ep5ki&C#5tu7p3G$yUe{o^wwOC8S_WxLH>i9&i5i z%}w1S50TW{4FrTQ4f8xwcFkqxu($a(BbuEjB6CKjsPy& zRM4Q*TWS2PxxM3Cxr()C7JtBN(IAnP_-XJ3{kU3`UsTnPx~L1|h(2zpq<3@PZ%%+7 zsqGTA=_TsNb@FQ5RR*79tpW?tr%OVkWAXgI2wYJ7;Nt=A-(r2os+JMLBy z2~-R0q<@1q$Yr=5rmQG<<-SN3*4S`tz1@GdB^|>#LjY-OWP3O4aJ~{$UI6WljvmmT zXieVaBXtH+CJT!n`+Wo5OPc2M2N_2)F-s7y`O^=1VdShlP3)X?#f3`8nOmop7d10^ zHBUWpL_gkO6+(Zpno{)tZAwVNJEQ*oV|_V>WLRUo-x<&9_S6TbM~jlYmFzpxX?9_Z zVoFNqi92M(612Y?yOLOud{XGu%CIGbrN6#o;)hZ5 zzua}^rE7=w9L>>fwCb|jK3r7*#Q-R92>{jd@(+%jxPRRjLkMH?PASS=Q}Pz~Yij>q zU|e#A-y6>_squ)&omfU;JreXv@<%+5yAloF^ zf8R@gDPjJ32K7ex&OJqm*}>vz+;Tpap^-Z#x#b`>Ms9nL)=H$R^vG<|i?J8-9qE z`0uq%f4M60^{o)@s68ZuTC#Q)7uWdl1C|x{Kk{OnnqXz^G7|dRKAIdKjhinvx>Jjo z;N}ro?_V4opWlUl=)-o+AV-p8L;0vc*j=E#X=~uhAbbimmn9^`CYO2;l0#Z z!S+~4<{}sR6@RPb8uVVF<%IAhfqXzUe&9PxIpZRw{Nd^tomZ=eC&a-Bq-46x^}@^9 z8}dDd_Mxayvr)Eqk*8q%1l*U*VC z6PuQ6cDVh}_#GH9$4!moM^X~q|9y_RtIK-N{poKvRq>bOCiOt2iOQaN^?XoisF8?n z-*<4teZviD>8V*9=D*joZ{M^o3UhXt7Pz$A8niTj$BJs8uWxNEM&(;0?KG5@8^u#U zFer+-vJ;iZ4D~fTq^3p!$2iNI(`P2HfrZk(h#`}%@LURNv3S$Vq z&buIjJ;;JRCFu2i%{nFRvl6fc;r{(P=*$yB-2i>(Z(5&Qj22~a1wH<~npwlg_Y)S< zBw2NjA8RaTB#Q4YdyH?7H?HQ6op(XX_(A6|pB)heJ& zVt#fpy1o%Kl%AGZEZWTH9Z$25!bs)>)qVkkG;wa;AMPKuNXXOKo$r6j>iRtguI0B} z2$F1aPm-j?3s^rcj5d}&dZ#N_0${tUpa(u{bG~Lc33pWduq9K#?5?F1k(6eN7fc$> zka6A@ltA8v_Y|^|?b#2sw4^dIMYJpGzksUx{;D#>pa}p$aO)wsdN{_6cYpeUUyr`xZqzcJy}dCA0_KMMS4l1m4lSau_;LSi5p@dQY( z1fC^)_i|Mbc1`jA?2eB^pPr2FWdN_zb0p{T~{Qh%if z*b7_SonewdL$$B?u_U?f3&wx7`x$oVbT!Iz+}W3yGR_1}>O>}N#pOR?Yhm)` z+(9xQ1?|*_JG1}xYw2Y@@ybtEyU@fAZWr@sj!aUZtD!iJ?*;LvYio~|O2!oLCmHi@ z=+jUJE(x_AS`S2dw5}DW`F0v_Pk9J25C-X)a^H~7Jk8W;yIoK3!axhK5+*m6+Z*uN z*vDauPos<|xhweUmQVp{gqCG1cEAO;LK*94an4FfK0l=1C_S(&O5P|a*P>y= zm=<>1N99Ht@^ZA+!N=o#(_fjQWDh%W-qByLcD(8B{C%1?-9992gM+R2AL5caxjuy4 zZ_Pyy)5gKT&&a6A!UOMT_rRQ+W-cOcscBla*8NcB1lS+nIV>NlxaiS`=gQ0iHexx&KS}+m z37?Pkc9F*eyQg(lk|QqJI<4eZR_r)8tP$^c~crI z9a{%+tx>Mn)hegNvGUy`ztniV)&v4n>yw=$OQ9Mg85Dsd-(VhlB8*@uR^*-4)+CYm zErSw7H1^e!K70VSPSl^&OR8ix;^L`TFB3+5-N5mTpP9)8Dv`RyI3P&~6p#Ohqo#hG z!fT5|LtIlR&gvh;-*a%+dm$*QGz{WOY1g&E)C|YIuD~H8uQ8KD;ur+P$89b`+!iy@Sco(Y(ckY3D&kz~Cp;Y<$TgNsWXRKCh?tA(nLdn#fsHzU-|~H_?Bx zyTzFkqKH*ig+6X!e=u4xUSJ&sPV@>qUV4Xz`?ExxD7ZwSFW&bQen4K3F{V-5&l+f} z$ZY?hJoYK2{Qmc$)LH%8Cks|&+4&0t9|SwQfgK7vN;%gm0nv{63y2fq#PPEkuIq))}N z>2w)hX6^YxhaLkWmKx{q%0^{|(l%_$$7qdx2vkR=^Ermp% z@`%fJ^{mcNFrC4X|JpLLG*)lxzaPP?EfN`((($B!o_pS1ThXgc z&h@{7YvsP_2kJ|kGA*nyVVU>J5LEA}y3o}kv&z^nVj#a*wj516uBL%s_4q#DlhHx7Og*3p>-)WcFOB8ePgoq{C_ zP{0bJZR1u2&!5Zu;;&D%@^HPM1qpP-WOBPbzbjlXSdlDb@x7zh$Q2EAOLOT={TYu8 zZje0X`eDQJqpz!-Bu!TSuyv^=I;xX$nL<#7spGy2!CU#oVO^2w^{J3Xy?2g}wu@(B zjn9m7Y&T8WWMU};d!14vmXB_O$zo({A^#*cx|NupUS5B0AZ`ZtOwZzd#z^Df=bUa-B(C zQ|`k+aa&dK=NzZa0A5z)Yz$D_b|7F^ODZG$@eX+=9pP)Nz5)V6=J1FRs&NlIy%B<0 z`%Ge#uRCf2BP-&iMI-w?I%S%n+A^Et-w)_hoMmk!Nxrjghrp+W)S1bSvT3X8HWW1o^@GOR{z!Q%U~tR0jURN zv0rax*w`Ogq8Es1n;#u8>{PGyGVK+wilm(iWLl({(zS>%C3Qu+b=5ugYT-#JfhR>N zTKdy?z%TWwpfQ%6JZy2xYuBy)kEbcsg;21gDX70Qp%(yc!k67~1Ul@LEC(1~yaey~ z;;1_JUihY7^F3elG0(@1;myo${t2hVZAa2@Zrq%6xDa|xQsuwmkp(cs`Vj8Vm-p@! z*28TaNf_Ykmwf($wKr}5Cj@V9R`|K=7)~5F-u^O_?INFg4!V@3?w6{gEM|U3Bfj^$ zR_S2&uRvL)$GjJSN1BA{m^(e)51JJdW87l2ulX6u-jfc;;zhbJsLD?{NbXTMNL!KwV{=v?`?Ey$IyV+#%=Mb&=%%6ds z@R1!nb|R$LoIBHv9Z4jIJ2*}V#yf(_SCBbDM&i;$ zf19OH+jO21^Wv_rnkw7`t}I5OG0cW69TvUf+^&cS5Q2zKQ%DIL*j{E9vmB%l<{=n% z--VqSAz)k1|73wfEcunm>rUiMYQcx$81_qKQM842YQyHLfp?n>8RVkw0uC)00~Pa^ z*w6E$Pi3ryGqd~2Q&x_ueWOj>`hgJtldoQGKWu$HxvHum%m0C?o#_tA+P1ELLq;M* zUEEI|8dAP`fInE3mw9SC^xrgy0mw?);@N+f{6ZuA&)?%38v_|044SN+MQaI9mgb*3 zQE0h&iYm4jRvuv?<0))O3`n1K;c&l_)*qv?l2IY{OnrsD+8d#Mk4pvyy&`{1 z`uzFX`ligXj~WS6Lknicrj3~@PyaT6(;G;^w1++&PHU)iYqB3RI^~rQ{Of?=2;Z^) zFAAIOY*C#i{ngTR?`cl8R(CelyrdlO#Afs~2z4n4QKg)l4%X3exbtSqJ>U7sKjlo8 zf4ur|`ox6Cuf$n|1C@x`6GNIxvOk8(vmm9KZI}vAxhi^)WOd~~xM8{)?oy{U9k({t zL*WY>*VRrO1KWCeE|1~&dNe}f^66*EZ!OTuV7QOL(TT$MtIgH&YT7w)0o?6evmZYk zE-#0a#i!@}gh*Lq#(eRlv2M-5l@NeD0(2U22l% zdKfGfk+wv`CVmeW+>w$LHuFOX%)Na(gwD^4JimQX(Udz7oh-!#tlE_}?}XXIBXs|W zU`KXU*pFhANPhCJr_1$fZAa<$%9#BvLjl>Bf*D9zkUwik%=vZKRGqo_LGL2-zV5ak zx68A)Y&!mU%sYq+B=U>&pg&Y->g*! z&mRLV8mvQuD$#BPP&Lbb+g|OaRR!GvXtu?c%u$u5^uw~#3T})J*Q=~}t)DJQBCYX_ z@^tv94@UFz{_mY4T7ScF9|V2HNMU-bH}O%wX)t;Ock;fOgh-UPgMO)XmT5fmTSO1fN)6Q8noYB5VOQt^gs zAWbHP&wUy!h{21&G+Y&b9mruJ!KrMQ?cCRAta&cQsLCd@wQ)(4A?qbDZ<2XL! z|6iOM;M3S*ll1CN-?N#y)&s7fUkSDz=4hASbo3N)-4_3})7I^y_Awenl_X4_k3S)A zWD;MIFUa`9@)}qfJUphhOiY!!>f#$0{{pxMI?kujwjA7AV}wrG$Fj06pG#U;XqjS< zcP9#Z@A!jJ%J+QW<9}^BxVBbo2A-b2o|Fb@7a6=O?51^3Kjf;dvooPadwII-1E#hN zL&mjvHrCd>kOK)hxws(uqekXxB>f9z47f~WT#B9Qkg-u2?7^grc)I zj3OOoo1c8p4q0^}_;Oe3W0&6Lmo=sG?)!aG3D#PTj#5ggD^6lAm}Nr_S;xBGo^msP zsqf5dfnR_dA#7TZu7QCkD?Zdf_d=_T%yr){lDz}-qR~TxAzs@z2tCTw>t==y0&a_{ z@0dd*L#(9j)L;4FX)Llb9N!tIq>mtPvz?5fj&ZTVGEPt!+>Vq!COaLQ?ZGN4fz0O_ z?|YWEW>=sOb1%uzW#CV!Tfw@Ju6q=OgM6cbS14FdWjG5iOsb6YeTf19*UbY!aZA!xp6fs{;KYE!7i|DJYI_y#HEjA!*v{qKWH=GO! z+_tRG8Z&I;wIvawFapHq5!Vc8KjmNrwiGWZCQCKs@4wt&yY*^m{^jxhcW~<=aVP|Q zE;67C*1x^ufwdy>o%xrJ6*BejpE?IaOgrH$$r0DQQ~k)~Ul4au@je?HKtonmGB}|+ zc^hx?MlRZk89NG>gMY(i<_^zxYt@v0zGU6aXEM)5D>lC@|9;s8Z&!h3E{a*ILDIE( zj0AbEg7v;IzjeSgj=#dNJh@)MldU0(roUf(`p;YjI8kbNW@(sI@v^?Xj4J={z)?Q= zNpNRXOnPAD{J|wgTIPt9%%mwFISAZkw9KB`>tUex}$OltNuG>1=+Im>Y zcs!ZwSY#ATs6@^Wv;o;>wi%NS93Yhu5gwHAPn1+m_6VS`vC{U6!)Jyf=yfPe3i&l~ zXQJ=|rfmrwD18+Gpw{T~0p=%UMo@ z10U^K%^;{oy#huM)X0h=ufEe$k89mY(z>HT<4c>C>j8A{yP|;sHPLqtXp4TCrVaPk z8;u1T6_76Z&yKK2a8IlEqeXp@WYrH}>-srdbxC{m<`lM(Po#wB4)AL;mAO!r4yx6{ zLTw3X%=P0^$+-fNt89DjHHbmbC`Tj7B0I?iM+w(4u$7hXYwgvFXyqpVBaMzGJ3P!- zSa^45g%@N^7m^asb}!a9?x;15x_suc1Y}J|N0v4JeIuz3FT!8F`HJM5q!KGegg0dJz;C%S#kPOWau$3Bb$ z?Lbg;e%`kW(ORmoVn7Jed+?OiWBQ{JZhoIdC23tZv~92EnWSOp&&vuQYfI8w{OhYN z?!NCU9><}V42_)-wt9xF$M3>UAAB(WM(wcX9fw^8&+nb6>ivafEsS0AsLkW@$}e3D z|2nk9WQ|rYPdegX>LkDAQLeHIx3NVgeFfXw988>l{4jbLR7r$@+nBpfsOb~APLnvZ zrhpMZJJ|jfAM5x%9Mp{vZ9QPb^3b(s>=DrMe&b7Vn_6wkz)A}=S;tW2KaBGAMRdZ# z)*T>79%j*QwgZ_K(+WtR^K){}<5bd?TinI(gD>uz#UoD+_xFp{&y?CO$~w5{qJnXL zt~^$${-HsdR`9W~#nK1w9TW!kVNub0#Dul@|CT3}FW!SLnKbWNLc|$kwGa!GTE2f# zd{Cj#gKr=ed}8hXxY+oyz_gd6j)C`npW&xeX>JNV9O*cnh{K7{BW?z!8ioHIAx zG(d$FY}a}D=r8noIhM{!owJl-zU})4KiEiO#R99$Uw<+X&QE@bx$~{WlsqWdKj$Ng zGY{GlzDe_^NVZ0-ta*4sWSpH2eDWf*HQNSxw7$ja^MM+11F&z$+o4Eu*SN7Mpw-4j-32>%wQMo;f0eLa zN#u;l269-W1|CKgXHu0LqFua4XY4{LhZ#a|`FNtUeAB6L<_~u~j;!iowTBaOnBK&eiwxAwiEvlsDeBXw+3RK9(mes+&#Op)$BRFcg5*7P-b z6{l|^t24pWukCSpHX(in==+Dca|{2(!itM!6ui?&i-6IbGGl(#m&fb{~B32J#~GY9X$1 z6X!JrS!V5a42Uz7yy<6-JluB$CqX6EkZgV34>ouvmESEV@^o_(6PTz7#lK7^2Zvz8 z%F*8a?4bYLY#_AGoBD4!7o7+xHMI{1w@_<5?ss1 z&!%CDsy?oJZ2LZh-MYCmv4SL+cz;VIx|CG? z48~P%U?%8LMKq}Z4OLRU?N6V3hl)c{rSJRUc6?&GNBJDJ%IJj$CpY4pnO$BhIwWy& z%N~ZZ`Dzt7+kDqkPB)!BP5Rc|28i0lc;Msx`(||`7QvYVtB+yp)l&+rPBWg! zL0C%LO|v$Yg9q~8_^c=*w%rF3wVAo@Oyw5)vQojU-MTof@J)lN9jSg*tlAo7k>rZ9 zA6zwhxK|dxHWYDbL_x&#rIjiKsza2&w+xXpOO43q?jdwiCe;tml-q=`C)TQ74uF@yR5k< z?4k5w#oO*oZguY3Ev!vJSX~?K^?9|;eP`Gx8g4;6Q$8hCOH(J3?=At%1%Yem{(-r3 zgXWjdz&8B$hV$n2Rr~%m#gfMo(i};#erg6jU^{zHCHddKISX#{dORMB8K|C)wo1}` zp@1v5K_Tgwcl_Co>H!S3bo@zt`E~uyy@=PdD)LWTXd%Jfc=>_B zQ9O$Eko{uQ{O@@Q*K3xt$$1KLY_l-EWf4W~3V}S^yzC?hJkBqk*VnVo{nW!REZl!M ztt~iOY1u4$IT+J%+y6U64Qn&zaWl}_Gpf579pCbHg}Z3elify2>)DsoTRfX;C@GhT zu{bj!sENf^1obhkPbTRn@{v22M*RkKeKWTuk!d@qQLJ;G4 zZT+r?z#lX%^O8kw`R$#ug}Gh{?rethoi)JCz(P1yUz#71I+}a z_eQGd;f{7Z-7J2{Xos1*8Z8mqkHYGD3pEBIcCXoO_>RfN!SS3;_WZEaU=dn*%4q`7H6!{=pgEXdH4G{A~O~D*JrD`ujP(HyF|rTjN@HIa#1<1iY8_fvd75(Y$Sx!R}_yAWt0h zVcgOKfA{KeuUmk_R?xr3J)Or>gT|NofP>$;gNj0K?e;;N*|diY?bV$3{0hy!%t_#O zR=mwS>pCOA5Wjap-O$p~&6Dq>JCsBtRyudZb+@X;{ZTfZ|HQ$yg}Kadldv)-V2GNG z$7v0$JTmV?4VDDu0Onj}Fv7P+sZBZIT3LHL5Q8cqv7Pc0j-Quw3CHs@4UI7#E`8-Y z&53EhT>Ln)2_XMkGerJM(fAy%8~6K!XEUn)mUHnEf)PMJe)@S-3+@fBE^*~;QsvLc zH1b5D_$_0d&GeIdzR05&7LhjMFuB9XHXQzCyC)s8mlaUUJTsT&ngK=k^A@ema^vSl z&lo{Lg{hpJ8dG)!z;n&0so4gB4zKD8zk!fdo(>5t$Wcb^6QtUfDafZSv?ZN)jdf*dI4YO|l>!o`*p2gD9s1IZME%`#Z;h%cApEWI0(kd1vA&cs?b6)T6&U5-&pQ!yjFlXa_*a%(`f(tm`?d|f# zV{Z# zs`mTj&xMEH$2I>d;2KzlhpX(v4f4I4v|ne%Uj|xxU&Q(A+gfiATgFZ;_nm(D6@EC)p3+hKf>;CZOU|~Zr%MJS*~(*!gIBk)ebAKO3k}8O zecVIl&c+-Q`x))fQ4zz6^6@{6PuRa%HH{Xj^Cl`dM7;^7v_#bV`4HWg>}f3mSbHA; zlFHGm+xT%C76E)NRv6GtOb8R3o=x%1+Fi+Cu6(x|)~E|&(Q5n4+xA^_pXw?BKggI+ex~T=6an|s-xU*y%FDl6kbg#2z;3Y0* zvt0M6{XY+^FI60v$)Vd=>CD#)zco_FJkwrVpd(@^s)H;Z z;m^;`y$bJ!%m@OWU!3&a!h&vL;+J`+JHg3PF&%$K$@c#KsXuH9cz7x@AKHLkwdbce zOOh6+oUnb9VFIdVgZ<;fwNPG}2rdvwhO)ll10w;6+K~6fHjk-*1OtFt5yxdfhj`t< zI~}aN8he>1b9Q!yx7VjP;gVe&>f{moM=N zv-}TxOW>FRhx~byr_&+X9FF=Y1GGMEpt(5y7o@L#Bp?SqYN;mG2wekq5_R$u?d z=JjSoj=v#MHo0nMe%w){=!kV-YU=6gO1L6las6{KG0;kW4gM`5RY;BQ&q{TWjpPSW zluuY*oomOLBtW%GTps$sDzS2maGtU)u!x zG0@&x8yK_k-_hfzg@-L@Ux?>XxX~ZCwnK z5)AM=B%^=FR#G5d%>#<#orxYMg_^ynz?A>A8BQF~{(#1V+QI*AGt=o*=b4>;xZ~{| z2fI4S@aA4rg<6j~3vtlw$#bF)N;%qj3Gpy%xGDR2oj34Me!2OPAa!zl>;bC3G+x*G zf`ZnPlKIW^-=y)fCaYja1wnYvs>8V-2(p{SS$=+dE+uW4n ztyyocF>rF5JU=IR%oCgjZ+9NgR7GNeg!iSrhx6?rvm|p($bU}=RE(D=O(q}(;nN?M z5_lkm*VNVqnw+<8FmGGk$}F=Ldw!7R=jdnt1gne`cIT*$RIJ>eu2>j}S6Te~WlTBJ z=n5_c61u!DjAANU!$!U0R zJyxY(ktn;XsKg%m5My)Vat_*X9XU@^2)WMmTFWD@n%bNH7Q^dB!s*xu6JJV~_rrc* zg0@?3Tbr}R7!aumOf~0qt$q(F)%Q)cYP|`%h%_JrM`?=jo8F$c`Db(%jZtt~~vuMb$uRIzOOH?}9r5)Ry2l;nX_r zaKykhE(sj(G>_J(vBi7$zE3D941kJ!vfNZvRmHT02Dju@MCn#sQ_~M9^)D=5=O;@` z7nHR2Y=1PF>#ko1frxD~wa?j9%WPO%hMvM*vvLtW{Fm%%^B2veGLE65GJf~FGwoOd zFG4^)lf??6E;7J4l60`#j$!nG&b~$9+F5^VTGZ<~jcay88a5BB<1f1xaN9o2P$RGL z{p@4-($MqC7};KwkE_Ox?^4k}L$XhImQS8FZ;MMrgMw)(qg{ZD?Q{jNaz*~rOC^F2;**iJeIXW&BALLTg%8|}DIh$xVqr07Z%_qt zYe$|#6KcQNfUcWCIruyS!DXPDI-_&OG^vy>bmUg@ppl=8%L}w9O4Ia&h4Cs=hNE=K zwjJe#g`rvax$y*4L4>q%;cvc#&J6^9L+~!JnIM5TY_SwJ$W2x}r1Cwf!@u zE8cJ@sGqsQm+~L`eIKm1tS1;LU(E=ztWuji9-(gF_AeP}Ri`t#s5#kft-7>eYNaWH z6+&|%jJ$<+*C;yE4AQe?K4sc%F1%!46n^)fgD9uyBLgckFp39wLUG67rb8i4y?GmF zGurM^Bi$4_i$FF12>kK5(a@jih3xh$CWkys*nhSJrE?I*&4Tk?UEn0pnuQcL>cMBB z*uMEXIyx)sOkBks#q?bXOA>>#?jO9z5peg6|$H9l!e#(-VJ#_MtdoaOb6=0#;eOkk^YMU_8pG|5Hi))kYk zLzZLPj%sz%<3ia)p91!QT*Hfu=*{|V9|j+_U4|dz@LakZYYJIUuC>bZ7?3S746u{K%(l;b{${>bk-9+RM z33h{NVT$R42M(?bDklJVH&th9-UOs@uCKVA`rQgHNvz!|ti-W?Ie7pFL`xbuDWtV( zKCl_}s_XYF!A~Zm`hNU-WlMMA0G||Z?(N;vkT-!pOEH=ykAl+36>|Kx#X0A(&4JB1 zry5B=Zo}d8b+DUo6SpRG8I|O@%SA~YKM+i1aQshQ$|Zto-4Q-w{S%((pOxCz^>y+v zB4#KyI1!?L{Gv@P)oQyWK9p0zt09~Jpx8=$(S@6hvyL6{kEn}({XX8|e+SExOHH-0 z&RN9Q$Ys$>gP`uMS`05}VAVWgS$xN~uNypaB>q!sta!Io8o)BO_MV>e7HD7 zz9;;3GktvQc+VF@1@hEh@5sDwvdLNqB6=BKH`QG1)DfkPu0WFt(}pV5WdjB;2@MuI{A&HpZ5*(G#b zQPuy83;M4PCj7b)yxH0*l7lDO#}`|`2sOKo57fH~;C|JAB}+YN)~KK%jG)C#CLJ)n znC5gjnE7e>7cW4=sqQXWI887syxgcvFdoW*TU$3ix65-gz z!ynJa1b7DMy3gCjVSH;le-lE+OlION>F$8Ba$mNH^3qc_eE(%IKH)(7R6MR zL1q++9ho6ATDypMN=#@JS4|uHyPlNlM9=@7)Vkx$c!mcl7r-ej%%x7wb?&@rQM7vw zYMLZNOic$%zTfcyAs{TwLPJ&dKJ{fb$RHNjk*3sU27UxuTMc}KcxpL=cAP8dm(kw# zUmVWu46gs3A;rZ3-cOsWPX{Vw-TK>9iB41(f*m&(tninB!_Fl-(TjptXZ%iot*ZL6 zgOG8-*!3XRj1@+ZWeJ01tBCn6-dYrx|NYaq9GYw>`S96T6(amp*bi0W2qJ@GY0=5} zo9hY&oHjH5Gox}j&n>jxdybYhK}4hlE+(o#=rVqM+vE{b9WphFa{>RL4)j>UR3+X2 z_T(B7fXN21v`}d$TAJ9fDnO)4plZ;%O}7K+UY>yGA&BF%wPMo0E;8i@hf6Ijas~3@ z5;k$5lhkFqsH*)aNg6UGtM%zZT+JYc#hpw}J9~;hVKY3Av@#=opr6*ao;$-alvG+Y zj?)j%s&v?#+A!>Yq;(n{52sbq5095MGbb?RfMb|EOOS%99B!5wyVeNy#n8vk5NF08 z7xiG(uIyl}nV|t6f1^kZfbJijoK5${B?~#PnGW$M93`ctwYi+{TwEl{XsbGahN7Gk zCs+610V|wI102b?^$?REvgX3sPCMqvfvXQSezo0?dU4E4Oh2`Mw(za6&fuB`t zAEX2^I0gT4(vrEmdZ*U!Mi$!XIX zT3udaZZHDQW+Ye8F^m*aF;pdnGu}MU=c)bp<#l?yWH;U|?9{6C-Gzsr6XD)p2*&p4 zBB^IA9!Qu6wN8&bcRCsG;)HWEa$9v$!5+RZ!Q^0sCTn*Fl&x0ArGUVa7q@rujOwvj z?Tnf@k{N5jAF9EqPQwWrJY+9WLo4N%e;*G`XVNANlk+y&wRkmLB|9kWR4or38VC*D>gL4?_ z&Bfl8c8OM0O3hI%ublp2TKeY+B{Oc*gEvIz0aH`Ushv-SGgI#q zhk}ORvs0PaZ2s2&+2-rTdB)U01AEfXulj|JJrlp9Xz^|DD-VK_ zUjN1&ef4mnsJWP|sC$|B6?jQNd@bgiqnG@?LTCKDIIBpI3-5XTtgKD75qYcHLw4ja z$lAwx4~+lW<`xJnF9%6wCT*7RpOV||r`y}V=IE53dcQW=G@rY%?v-6(c?JS!mRi)h z4ETkTU)}rmwph$ROLaQqi$yq9H5rdT`dVdP3$HXXRW`B}B_q#NP7~!m>0+w}KO6*= zbkTdLFVo@>TSFk8o_sS?Q)|)76msp98p^D>qeQ4KPlCK@AVqg6P#9Hnh~M=@dsY#yUuRCF#YGqXr3(o|PO#pNsG|qj=7iUuM0_ zUn5rj8=@B#9SYnJXzY$^E$X@^+Xo1PK`D#mciE9L5A#^z6zP6}tt^Mqm?tpgz{xVQVV3A5nA5smgPiYU zaqNPYQvkqTX?&i{32{UJ02lA5=#aUju-U=n3S?dt7h76K12ev)loTGj4P7azAMMAh z>geGq%6G}@Iw0AI34r^4%H!d*%NeIO3)j6Tv*_%nP6N!M3cO-~qqW{h$bj~#=lT8UX3jIDun-qnRBLr9&c@8_ z8t}ubs-%p>VrG#;^yw9(ji#4kW7}zLx*SOQoH&o@j?H+zUmjlXyHKl@atzhT7)v^jvw@zTQI+^OL_}d~#5i)>^oEmn*3MB}OXYuLndMf##=xYU zQQ#4e%&LoQ9UAYf5fh|g`}Q9gU2&TZr!pGb)%FIz~5@vRQ)IwtQ>vT)uEk-@0=7#Obbs+Tbt9PDn63b)Hd8ZQa)oRcfL_^h%K;C`F1~6r>mh6(cGjy$L~Tkfx!R&_Pi21_&x0 z6p${xC-kfICLq0EYC^9`{+s99`{DhRk&K*i#yDq>ea>2QuHT#l6h$qWKPcPsEjuYa zHJ%J%*!b6eJZ{zXg-DRvXjy@Q^>_P6q~>1+1Go7n3w>blP0_2bo-QAcZ#R&*Id?E-I>RSr0gZ?*!!YxCkN-Ff8BoeBx4bHqdgqRPrht1;0n88%*R=090SPh!EP)y~z zZ&;P6r-i?#W7;V0VwYNm%l%e+S@xIY3}8~bP1QH5Gb|USS9K*%nr2=0No%ajj&8~r zVHW}&Vt1qPw4wKmFI}N+U19?85GEG=g?HR;*CgUJN-=78V3}DlQn|&DE82Yv4&UU` zXC4!jg0;_`s*9QXVHJ0E4Reqk{U=93>| zU5ZC>+X6ATw=vMzsA@lud4z6vIyhj&baEhm(TUlqB2hDk4)rORAHNvHs>rT>RqA7O z^F_hNAhy&&vHNRfxdyHo!cUr|b3D@7AabrL)HnCzvZ z5hXZIyFm%5KCmrRuHi**Cv;fUdA+f2N!ofW5iZRcF{;hBi7;^K{Mxw8@*Z@-?N~O3g^3TqD zW{G^FPwUoQCIg}8%Z3^w(tEeoRmMy9J7*VG=7WHVH~?JG=c{2>I@SKZ7+c%)wM=@( zaVntM><;hiiNXAOM@Du41H-|>s*B6pGOX3|e48*gBhRgEOD;rG> zk8}nw<}#}jC#@F^_ednLC!!r4@tRA|y+IqR)Fr(fTv3#Sh!7J`RX%j_9*mK;I9O58`ToTW$i`}6{&Y`W|BV_-M?{RWS*JUA z@xJr@p0+%E&><_bx5{SQD+C4EK2|nLoU`yer7ZOfxrAN%cf{6Tgga_a2i#TpV%p~P zCB;GFIF7-YFuWiI#~f#se3lq^^wS@6@ks zds?2y=vG4F<&X<6_9w%9FH)ObdFpy0_aGRSW+u(NpR8k{^sCwsEvDyL*Nh@Cty#j+ zWK&3to*+TLr5o5)+<-a@XUyHg@Q6sL)O`xnO5-=_RL-LFHoEwrlnG?IGMU3X6m}@n zMBeN_(1d`_CHBL*+nSP&4I%1)bnEC?I=uRwsvE6Ryamj$JY`0e>f$iRj}S{{=DP(a zH8OKSR&ETkjGV3z=)G%SXZScGnPk=Ulful;=5LgpWWzW3G8@}_9rqUVs;Vx*JMm$c z6wL~jYF-@rB!2TJJY9VH8ePnhAi-B5sQ2zV5=0>5%!vnJVXWb*f7;h8lx3$E4GnXP zw>*UfU5_4loJVAm*91ROlH*Bf(TEbH&70MI3i=w;IyHOVICs@ z7b;Nt|1}vOzoJP_PPR~EbdT8YFaw?Q!fAQtxryHOA@wD zgH%q-kr$Ex!>L^y?x&^h-^$j;Ed(813AhZ8REN@oBIN3{Q>5XI*Qn!OOXfSlB)!E0 zW^N&a)_`kD*NLX&Y}-1;&*(6zSFoY$f34unXnP|t>Z=}-{?Vwvv`c+u_Z466iTs_YXF^QZ+}ki#ycF`5&6J@f37SuJ$tz%v#|DWduQ|IqSr`I$*{sAjRd^G_Fkrav3LU@L-YbIrbH+sQjAUZ|RRVN)hQ5vgdnR?l zv`X|f;@(KG_+NQXnqnh9qn+-ZurW9{T|7J0uh1-S7UG%yCpjW){eB3`b>LBBwno*h zBs}1}kltAmDPCpSw;x`tda_wKOz1oITZqAy?Gh(_t4GTnLrz^R{u7;9NRWFD0?L5* zvzEsPOngGUgR}Umfmh0MJ4MMw8~4>qWfyppC_w4~7N6X4M;iseovq>HATDrz4Q8zZ zrRXtP+K=sTs}-%YowA&gwpy%cS2}4^L+jSSG4d#Ia7z;}dBG;RkZIZ|iOwdwcqG#R zys%|Yk*%}&!~m;$X3pwHg=NFS0y$o?5=d-SwcbU+3MW(l`BST7GQS~KYYxW_flJYE zu1$Nn2fRg4fLL^3ONtIKtS%TiTJ@!^^dhJmg)Z_wt03+Bzv?$2vcBR3>)JHSFGck7<2vjD|w8_=H?UAZ^oGa@|%H*D!CoN82NP+_h(RNsKpR#A0f#H=v4G&OP=gDAF za$30jGnc}O+!0G}yn}xHSk^++7Z+P@J^GM~Ce>|^f~hJxBrc|G?HHSUVpv4-+6=Jy zQZTAB><#?#c?2ZxtDXd9(l>RguDblasqG5X-LIB3RG(Go?@zCDtxecCQl(7J2`b&J zWSu*-@Z@Yo0o;p=Dvr^f{5GHBDTLld(lK)caZH6DwIkM^8U^P(qJ`$_qSz*b5ZGB$ z%-ngh^gC0Zhmyrmw}Z*H6{5H-r-1semBuUS9ure8S2ec)$cakX#2ylOu)f}1^LrZ< z&U)olUo#wg$m-MI5}rNAl~uYe(Y#g&eVmk#YN-v@$%*1Q zf%U_^8GJoXWVwdgEYNU>rA;AsYajoQa3#On6JSZX^E-uvEPZ59z#?g9!Wg zckDpGKk0O*c*uB`11#!qa(=;)j$IlOG;O>8@)`j_h_d)Ht8&u=7Ag z@H#B{8U0MaV{`VYG2YtPXS3A~fMtWSCQhwY;f)&GOLA!ED;ne)6~qW_0!(^p;uz~F zdv?GJcq?%t3-0?r3`|8OV``IvKq}iJiX7xex^@4RAMa5b?*+IHcPQs`OO|j!NY7ms zfVK36$O>{QkW>}Vs6Wdq{NBuV(zGHQ>a+&pqW&$}0XKo5=;iuB1ZlCVx{foa4RjGe z$fjdQ`>Xv*U`!ZvlKa^iSP-t%`y7Z@g z`(lccgU*G#d-W#)kGJjg&}ffjnL`It-f=4FS!`64aieMVBJXE$rQDcwgZwy zi5(N!7Gmg{{=9M}R>1|DJ+PNMQd`=v zRNeh@$)GSA@EsY`P4LtF_Q46$w?qV?$cWBkR<*8P6zA{LkvsIGI~E7z=F*bultk<{ zr8#0Bi8~z|r}fxTSFrVz%7j}8$&p2ufS1FoJ!Sk7lmqb4^4}X>kVx$VurKNo?a&46 zS~EqD;rC(XYZNTvc$nuX;?2@y*un3Z1giCZH8$p|KST{Pb`DGJw~oa{I?fy`LkZiWk=*PNEk4%nA@~gO6G^QBjnsy-2q2&JBD;tjc?YE zw;dkeL~`EwIMI;E41ok5S9{rAfIuKL93Kk=qJ90fAOv#v$G<_~r`9b_cM5CgHGpdd z-$XGFj4;+HjE&OE*EZk-A}4!8K}uF$>Tg*+*&9mo3QDp-`&3ktl||Bj>->KXaB{JB zu=W1`AE0Q#?+OmMbb1C|7h8-c>a`8T&e_GuM(PbXnT_)sdlw9efp7=>34(m^?0%t| HW$^z27jX;1 literal 0 HcmV?d00001 diff --git a/doc/images/fcn_schema.png b/doc/images/fcn_schema.png new file mode 100644 index 0000000000000000000000000000000000000000..fce8add9fafcdb89364e3da47f865cdd7e3b22e8 GIT binary patch literal 22929 zcmb5WWk6Kj`UgsPs0<+>pdv7IcM1s74&B`;-KBz*AV{ZlqjaZqw{%Fy&^^H2gYSFJ z|J+aax<5G1?6voL>i6U_L`gvk7mE}N0RaK`t+WIL0Rhz*0Rd3}0}1$sCmW>{0YOva zt%R70`|N(It=|N1c?~A!N}~e1VD|S4Y|Yp1Fw^FHb&RNiR_3P(941cEyk6&LkD|ZZ zA4~`(WttRCRyWx8PM-x*E?729miI$$RaNWR5rJnT`2XS)5xKuUW4J{?C~^4241A+Q zJD7gonl2Q4&tIumIEq|_|MAnpRfS&aEV}D1nsNV^s;VkV-%CUvL~UptlW|Kycqbhy z0zwomKBaJ@<9gp@k$RXPJ|5nHKBbW7+THceWKr9B^l3*jkG+|YgO^vsVnEmtb?_dn zhpXhhixp+9Kc;gl?o^zxiFUK!z3(Wp-~G*!xsZd8Pg78{Ka+7kHc7$Lu!d)0?*RT# ziF0#vpQik2IB1?*T4G6c6ZX3k4B4K5)%x6=M~d8O*n6sXFeOqZCnuNbx47Q>9j$go z$!9E@+Bz;brsj;@*xK6eQ{G<&!~ERb+L-gBn0dZetgjkE(-wa_3bI$8 z?L5}?JZSQ{QNy42At*r;CdvXKAc)?@n0bJiKZ+bM25mm~iampUYzCulZf?fM$6r5V z!hf$Bb553uWdP;TlysA6t1urvmHiedQRaU1Sy)ienPt5kk54Q69Casohq!s9$>$Qs zgYgX*;5p9I$ZjbqDd_sp&=;rYCW{5Zs!uJ#T8K(>6%taCM7B2rlfuPr%Q+L3{61Y- zTV@vcI_=ab-g-~&QAhpRdEIuC3%Y!qphaOcmRJu_@@%O?eM>;MrVX8_v^@!E?JrcG z^lCX>Q8JxfLA%rixd+yXoE?VV#0eZ)d>ydi%RsC{OV)Oq`pGNEJH$z(?&H$*lXt}5 z_sd_wintvtPPPZEbVZP@+Hdi0j*pM$DW((MYiel~k_O%CD8*ROv6#|O--`$e!j9L& z$j)>_b4@J!0y=S8Z+o-t>g{3c@%j`ZzKr+rPW3HuzIT7RBafY6lYBw~nBG;ew+b50 zU%b8iz9IPu)yW_v?eGNMR?D6> zDsGvbh}ktvMe6u~jvq>tmYOQ$as2Bt;+Vsn&>v=*-{2&tiIQqs(%)HkzpI&*=)73x zw7Iq>Vb*mFQOKzOv&8tk!l~e(ay`C!<9-cr$v6-p4rnZ?;|0MHjk1wJVDjf1OY#mxi z8n&>&JFcO_|1^Q0XjhrZ@gIg^=G9%Vpf$5R>yrC3AnON;-)`@Yq&!$?xHw+#4-3N% zdG6c~HhVi5*~z7v0gGqD9kKgx)5sS~?&ax8uUB)xbvD%&Wf^ylvtBM!7Gx}EWR`e5 zCjaM@x;!Cu*D|G{IJfB5-|7!~QWO{$(tCTbFw0$fug>zya$=A zKYJLUexK^l!(F6#`lbo}==qn5A1O5I&^on?^2H{gk=%==4=$m%T}5q#Dte|5+ClZ| zilOS$Kh`1R%vGjY?8$`oC!82MyeEVkkDpnzs%THC*_Rd{X?gVnHd3bBKy!qYV$5jjXbca%Ug5qG&D2>yX;D1K+U-mUon7P_}#m)fm7(7gW zrRj?G9$J%8-6g;KriR?ExUk^2zvGaJ`O6WNZD)L3LXAqd6WaqzbsIMDKPl{4OM+Q1 z>SEe2&d&>9lOY)&6{C@f5ZyTX3qnJ4LuJ-CXTAgIitx+m32(>x2#smM_s(DZj)4$+ zN2T!7=tRwdC^a>;xcKF|DG|TM_egVY8b7qFAJTPh}#>(~vr9u$so@Fpo zvBy5#vhe4Pm=a(vB0@^I&AsSW$V}y6|hhyXb>^Cr* z=Z{Q&x2^lcG}_JGeJk*%yaupwDd-I#xDfDbMNV ztm7G zh9E7Y^Q}H=SOfc313JUi@{fZP$_~YC`=4a&OwX%GE-}754AB>7ASIelohZ_|T-P}_Z%)`p{Kq<^vcBH%d6q+ca~C`K z*rWZ}R4_%4InH)UFC%slGrQqV8FZqpB-Y)%;x;2UL+_b(L{6;o>nn!hNMEt@rM zlq>XwI`b3H4oG>qYp!qBrrVb$HR~?Ca!{WiJWhR{`~r!zPSkUkJ&WZMZ8wd+6zNmJ z=UhrW$ID=R%DdzEmL6nUhmeSzc<$EAu?@WUA*W;1`N{H#4)nVQrHhr^M_ z3)#SE{lfkv z?qe?o({%zk`9iRJzOh8p5N>|Ey$_8av+?LQ)l#ta7f8NqyyLs>xb^`fs(Qckn<@?s zdnKZiw<v2b3md zwsFSBTnYo^AEq{ZmN^;PCt^Unhv07?#XC7>HHmB~8t0sbJbzEU zHV~D6hDUpy%7dWY6~{pmCcv2v>NffY&j9>qysPb;{0U@UqqWBUG?fNDd<*=5a7pn0 z>7##|qp$DX*{-_pOJEVB`(G~%>uwotiAj<2ygA>WF4NylR|>sv7}#7#wxh_nFN|ao zvRbsUvn!VrT(j-T$(^2$E5hM3_!D-)iSPrOSnO4ZIow+>`*({yDN>OXx>u<@_KRBq z)#s1q4iBZLrl%LW`k}NSMkXd*)vcxVTDJqeqvjj`6muX352o~GH6QP;zX&VCu%9vA zcw^?Pe3nQ5ChMJ3y8+P!!quKzj-S?G-A{OwD?xM`sHVxlL4iySvj>!-oCt zPPWJMpf+WCO~*Gfc!~8FdKviUj&~I*w;P82Ti~M(XYpMRFJbht7S17knbXVZ_R{{Y(??o^Dr6w=+4ayC5*y*_9^2=M;^LQ< zb#GFNG=fgt^FNUy_3?n$bkouh68|FD$gOpZP2xDI=yd^2y&C|c;he#tIBaucQ&Up~ zw|GqJq#c{Jyz45v{>}Y{n~{k{SDsR4c>6#GljNR_i$9F%>{p``6&PTeKbbqtT)Je><#`1CvH9w04%&eti=NtRg+1D>W~eU&hAK&6GI zTBkmWA$VBZVyL73&tOGg{Yxo8rsLPl^n{4A_SLDWsq5?O5P6Y-&EfRjr)KXlPh(Kv z9w{gv^venGjz2aVNeOcUk|8$xGs?CiUm=*%6WxzJ}T>R9vi-GT<3q67(6qX ziZh;5QbL+rg?G8!_JMwv+tKid(A?Y{h;43*r}l$OCC*F259}UE!80}J`|l3F-WdYk5xED7W9Hdb}PV_(+zs>0W?_f9f?j@uqOX5ABIVRbRY zItv-6LDRM)adxEoRyTJ~ox9(VW@V!gH~$JFW?^P#Zoeh)MXB3VG+7#2lOd`OyeRE-Dc6*Bi~3+$I#H?P^fmX*b9T zZaBcIoKNvqJ7|{|GZ9wdmoFgZHzW_2j?^VK!9rQJde)}%m}6V>2PC7PuIIi=CsRA0 zCGl>uiy=m&Hy|%BFKbl#o;+sGX6R3IeI?2p%MzAbU)BEGrK$wINuEZq`U>N1Zw&2* zRNY)h_C(U8i%+^7*Bj-lVO6~GY8z5Szht;s_%(7nr=PC zP>JQ;(71cp$#5xSU~^dINtV<1XV_oxt&IAQ)pg$v!L~bs@jLCwGnl#o<2NjD*}VjD zAjz~4;t#Y(G7ZRxaCdBM7dKp%#>Nt=i?JFkT)%}_El`U1elz=};if9A(9ls?VFxgi zd}ciq^sGoZarG|>ZK1MvPrkl!Tk_e=1+j1R>MwXEJuP&}Xc+pD>Q0DDYx?#BbB+Gy z*RBbQ6LEx7i5VC(!K}JCcx3(CpiP!*^e0!=7ecOyh98OF44}M=XFcSQCZ?>kfnq6# z4=yNN`;PRBN{i|=A&*e2^@p+^p;4qziS6mP{b!Mr zx@?mw0yU;iE8$GZO$Ic2ulQO@)1WCon5^f*f0~}mA8MBlk9dT}iO41AML2)?X(s}e ztDmLhA50}+9o8(F^Nar2;92N$Q-!l>yTBC~lqT%sxj$RAnr~F;(uT?r@lc8w!G^t` zrf#}?($9>5@6j27AsZa`@|&DVlhp79s?+Kca|@wX*Z6%ljD#cS1gB&ciF z-1IWnisGeM{`KR{bf_zclis2mxSsv1las1%6{Aoy6Tbx(^0letWH?`E=25Mbm9n9} z&pY15`w0Bw2_}2uB~X%a&M)~j!!Oe`2ozh=6YH*Iw%es?7dz#iMEaF1ygKx=X~5f` z=smZNp4%Z@gZc(|4 zwOefL7C7WtH+bZRE3YIkyYMt>9ybo;_8b|r!I6f#0Mz5KyNN1S7~cbA5;?o>Ia$pu z&sqh)1=6Eu?I0^-zi*iNARzv}JvKIWx8>#c>?dvCiJ6hsM@|Lo#&cJ-sVU0m_)!D2 zOJ!CuPyK%L*g;1WgajQ#@-jh})jFOcJO{rEQ9YJ^^7t+nH%r8RCUTN&Cq!=-qiY^9 zz?msdRCKv76gHlZq?j$|bl+KCrDI|qq11-vK{BdhA#>9|p)1N<>R3oq3n36C83S3W zms^0KTn}VR=0ryJOX*QTFa9z0Fo;2y!oI!Y>Mxf9JVL!kVMmYVElThwsv1sLeBHmF z#}F5QfuY`)!?Lw}RZ>u!3l)t%ZZ7xb`tYTuYaez?`#q!eJ9w!uE zh=DO~p7;jyBByw!e-x;mu+7(sy0HHIky7mIO~|znrIRF*A*xy4>6qEKH{jv5JJjnL zUJl=5*lJBN1DEIpC{!IQY01&4YMp`})sU-XqPQ?<`|!ZlVxdBTHP_W;Qi%+314q1O z8O4BGKnEg$NJ^2Nq!DF%cX<4{g?e|nQ$Bx2V_yHdN(A-2aoXd@CJI#Jc0SdGhI6WD zu7p2Qa1JP;!z7i86n8s4YsKgulqHtEO1D^Ih`4B!4wIU@ss|&pVAvugq2V?RN%>~#)lvraGsDhVHXRQ zH=^z-sh~y0bOx4#S*(tHdmf_75_}pHft*@JMl#O+^ zo<29b4_p<+91su^ip5P?%Mpmh9^lygT6UR!e?`gT+t8Ize#QSwl(;tOdXmk!(R+Jm z$I6oSa;u~Trcdq1#MCbcE-Et{QD#fztq&aQJkZtDWCIuEI2nm}IUVgrja_EOOQ5m4 zWxN+V^I69mRaSj$p)<+MSC@>DRDnA5r?lC(VuI3jiO|(T)~}~sMI;1rbnxhY(dmte zow}{gwF4WSxqeQUhUZfK{bj_co8|IjYJJ@^aU_bp;vmN1oOXN0dDcABUxFmE^2Ix1 z?ODpRR3N8de-1_UR5Vc)-ez?N;E7wUbTo7}0 ztrm{&$H>4A9VFJVifD4hJK&Y2{bm>Sh+|K==?>?esgnH~D_0dy`jT>ZK}b~pTcDk~ ztfwSHgJJB(eD&u8>WSPUm!66Wjv4+r70>A>imL1tEpM~ES`zG_`}>3^6aL}?XM)hd zu`Y|cZlCgr@3^q2&U{~SnkZaO0)Aejwyh_5O+L}Y`ks!C5Yv9uq%XBrU7y2k@8Iby z?)|kvKildA(zn3Fjam)2vSKf_8?Ky3@V)~=4%Hl3`n7sXjEl4#AHVBjW#;uO2{|E< zU7ducH$A~rlY!i4L^x2L1`hq9PYlEvY|!v-7C)CzTO|Q~fZWz&(YD1AcdRNW{(wpv z73O5Z>0`k_`OJ>m){rpHRCDPayaz5>Ma&wtz!-ZQwk`eJQ6zY~GmqVH0h`|RF4ED5f#k2QPsfy*Q$4Y0 zKy`!ODfWTnyN=tNi!zs z<}K@9M`ZU$k$yOJh&A?0LwEG}QCVr!Lo>; z_oY%?-uw&I?f%9kpi^|8cAH%>SI32`ku?=F^ez>BdNiKO|~tL{KgZ-o-)KA}Po35zatN zOyrnBPvDRdGZP0sJQSXu$?^9JQ|0=jnK*s#6&3}Hf_rQ%Xl@YTG-V`jW593Yq)Nt? zKL?cPe-?_KBhH|6QgJu|cRRiLzso+n&6}(Au+T2UH^+A+KU*ml$8*{}Tlvi{)8pGIq?gwG|o6|sK0+253gh0 zroQn{Kq8GA{PaNK!wdJ2|9vL>HaG`4;WYoxLWCJ~6Uv8u?jirP&wrKy`?MS1_-@-I zoFer5{X}pD9^XL8bg*$h9co8ipPdCo?mGfyfYcGTU!X_IqR8wV=0;eo=HCJbg3OGt zl;8%;pWqA`jrzN-+|1|yaG4PILDlOMzbFBOJ&32kf;TGoytA`&bLM^1bnl8Smw#vw z0h|HAQwGTXkBE^yET*%1e-7f?=OBKl@B5ej4Y992qH>71JsYoMd{|S0T|B4ORDi8}$fQw;d@(-$d|7aw10NoG1Z~;lwzGnj z0O1wr0)XyLJrAxsyLPa7nq$DLK7%{h!D(gmKx$9|k`b1LsM&^`9O6d!gG}Wfz)2K6 zMuDF63BaN9x)Hc=AN25n9RM8hj6a5x`5+uPW0bJi$&%n|0p~2D31yVe182+e|KV&o z{=b}!{x8A)|0oe$k^epuew&3TH*pf2!UvK3yAU9>9RFdT=l|~WpJl*4Yb;Mjs`V&` zDE4@FIy8gHH>P$wewuy@T+uP*;lv-$5BY0yMp&N#Rstr+hWd}mu>o2I>=-b)0*l`r z9-CKJS3q5%#rHNe>kEW+d^kJPP=rO^a8!s5P>{7CvwuYE%<=4drXl`jY9xhFJQJb2MH*!qqZt}Sq5S@^F83DW)LouUQ)x%5FMeVS%mUdo@J z1~h)f#4d~e&l65qYxER*;u^R%gSvjt$7x%(iW*_f!*Prz7vNlm@n^8KDgL=;6ZWPt zs2jSsSGzQax!~*7HZ~T$8?iSC#}N--_|jya<>h4q3vqz^4@?gW3Jnrk`k>oQAt1sU zJvliEL_dixw?HAyr*^R3$jwdtdJV|kJdy$C-S{P9H9$vF$ViQywC&u{ey%s-dc2kt zrzZu__tcR8wPc5$2XQ8zbjm*9h!$S2HhEpVSP0o|QHI?wG`Kkq4GHBBb+o&=x?-^~ z+)PT-lb&myIYlpJNky7H!Ipd z|I6%he12!6RM8(zP5Tpwf1dY{&dN2T$OJ6;$Hc@mH8p*N%vfW;0OXGN(}GquqWcfv zZu{?xRzKv*_|l7 z$y9*+_ic0kV5A9osgY72^rIX_Q_2Jgdm9@Y`}z4{{8!sV&)Zgs6RX}R9p2sE6e{+a z?-U>d9U|OE_e|3$DCPL|xFlaEzKdct#bKHeVklwB*z!$3dJYFTKnb=;njQ>L|Bzyj zd6eqS%c#Um&`MDrUIs{oi|Wshz&5+4W&>^RH~U7*-G7coN8f2k<}nlH-S=O_>@N-5O0gEw}DBHI>@KD8p>ue0hqfz zju!s|V?$nV%B!n$d89ku9W)~5GW#fO$Vqmfo>t5pvocxt$$_JOe1X0GnEwyDv6)Z~ zc*TKLF_9#Pl(2}F5JC=kd3k%rQhJ3Gcs%>X0iYm_TK$ih0C+0L-(WkKAS0J6k@1K8 zxJ287LN|!j0G7k-I8xt7RRYjW&GR4V9(4Y=_rYU8`UMZKD0r(7Q8S1z%7?XOu6QmS z83;F5u2dNSAb9Ys#y!FbQs0crOaQDbL|gm&%olMDgTP7*N@f8i*W@*>>HjL=e_Ugp z&~MYBS;|C&oQy)mE8BS9-zC$VCjBwOubyRqQ1B=pMojAP{;!(|obr)861jOv0Ng<8 zaC2^p=`q-w5(2jPI>f^)2HpZnXfgkl2|QS^v0n55dI`H_E$1inS)gy+q9&;}Pafr4 z{D#`nT!DSUETzEt{DOH_sPMo74kkC-7=*ZUNncO7)roSbM*M+#dfHlg%`peyXg>yX z%ZOc{OuJK$KkE6(+IZ0m#Gi z-_p*Ym+<~9PnjG(b_Z%9Xf`%Bdj|)bhCpb-dC&!PCeG?6UdgnHzIgDvaNTz2e|;{- zz!8m(PjSk9NSH%4kD%GypP>cdHuBN)uqChe?~goObq-t{-L-Ks^w$Lo0df6x!hoh! zmb=p-q8=8xfRVayk4tA(U`lz{YZf@==h9BH>0&g8@ z!r$h$rI?-CWrDUXZu$fAO+b^5y_skV18iwAlqs*AXn(a*S+McyY3@eC=fikUU2&oR zadk;$x~GB}Kr)hxxT>4o{6bz%EQaCt$uWwpCV>kU5b%Zo9vx1g!Pr~Lj-ZnvptPRG zV)z~Oi!-B^fchP`rBiO^$|H(dI#jlFK@kzf8ZAUZ+`pdG7Y?{V9MO07gP$Oe#t|?X zr1=p8LW9oiDk^nkz6JJl0I4~3hS1sf#L%F>9#-^Rtb?w*y}P!tabT{BL;)Fs52$fQ z8dKgnBop+DoH;o0|+KnP-H)d~RS_(FK<9`X(lgRcG8sDO}!Q%9V6!wKk!3CdRwfFzxHwdCZBF6;j|u)KdeWZxAQWOM_bu=eO&Y1atxYU*LlB52cV`?k?WkqM zU@HZC-cKq1Cw!LQ_zJXdDBIEY#I60QOY=z7MCID#x)SoqLLjUk_mJU12%PP{aJJ8$ zC?pPi2boOyJ9l8^TS%t@f`JxUAHqYNC=IJw@KWzG)Vt0EbC~(Z}RGy=dW~ zDDbol00}HscW8ee0U1FA8g;$yiy8E<@iw-$^3j>^u!1r`(n)U%4r`j57l#D`;Gq<- zs6O2}+DFN2mY*Ur^2?WUH>RK&|JAN2E@t^*>*cRLkDOv(8D!rhQObyg1>4)&Hu>Jx z2S?^6{0B~epx=bjB>gNDqLhXBM9pC8xmR;A@ZJePu@XWS7dieF63z^|d?vYauj?DI zh4r)Y%~)`cFou3oc0Yh6oFen{2tJFALI2%Uy?k&D$riMip97Fik+XJ zQ?<_59f1+`JM(k314oHM+M0u9pz-%<*tbnPnrK0MqIgQpwBkx|fv$Vio^gZ4nXI<;>?|*)Q z@DQ%EXlQH88r>j#L;8iJ69%Df^~!wGazjIaP)(j&!zDv(-{0AC0bSU2eM<>fQ3TJX zR4$pe)|rI`N$}?*24~Y+_+;Bzo(Eqnt$$r8>I`*WrBOdJ@y$b~P%nkmpVyFPd_EtT za`-1s^CONm%4@&47q5EA<3qh203?>|6=g(;D_8r$TKP0 zg7M<|+OvM0~4aW_*X+Cb{R4B3!Z zZ3}GF4J!yiYtjwP_QaTA-E3rtl-TiE1AHn#|0zB&`?c$K5kGdoK< zO*gMT_U`y@Wn~3uZXa$TUBvqOEelw7pAste_dsWV|NgzS)KXhZl7yjJ2m~_k#K8#{ z$l3OgB<5i5j{ycim(Rn4FN4&X$$Iv#H)qW9Jww;zu=y+~tyMm4*yL`jsfu?0){NwD1mo+|NGK zaqTHg%TlZ@nColy+2SV^@_w0~#v>2(eFoH#`w%UI4Ke-Reqbcn%11rzAp45XZlQkN z?)(0}?E>F7HtbVE-w8Q04d;cKnHeCPZPG-nVP=JAdPYxX@(f2Df2URXK%?3 zx|YiPn5HFH$#r(ep^JE3oRqIRD7eKitqikkGOzMCpnWb)Tmf)td` zp2_d`Qrj`T9F}yIu7!v%6L}ski-khPPMQnrcVW%Z9_ICNyj7i6^lu76hzPBCShNR6TO&a+I|t#4eqaJgDcl}29cW1lVed?CM$;cHdSaOefPh!@m>onFk zd|pY~#ene#Hq zL*{ee)F%38O-A`qb>|Q5Ro2r+DSrEA173w0P0Fhn8z%kmZt-`iI6nEDLjSg;G2~g* zqs>I9S&aSmJ~fAB*cbBXaEPdA8+j}ehx1j?_h)r)B}gigNwyWQ{l+%sn3ZQEpHCQd zN09NHmc{5r>*q}p+_y5uBDEY{(-Pm6K%)R!LC8UPY|^M#@2)-6=Pm72IDB$U1!BAn z@fe%1>NPexJ0%^V0ZZyHth?p~5wXv13b)=~tT;1`3Oa6{_OPG54_0DyveMl6xM7tW z_;#@;^RxCQYfX`k8q@b3t|upSKyh~^*!GFIhvVEaU%EWA+6o;|obMFC85-2^R~pLY z!%mCY?e(|+F*|mPsGwahGMrN1f=RSckE1d<`36jtkzt%8>et#h5xa)OS7a73oqa(a zPq7_&aXbfb^i|y40A8xu_9mBQtO=D&o6b$)1X!m3=B%Tr@Vf9ahh6;4&r!6xze9+w z331-}dzbE}a9N~_2l`VwRBY~!Ue-|4o!-8kogJW-0ZTSQdQH)i;# zbt!Jbwza02@JRS1xIzEubLV7&=(U#9ZiI7u8dDdMNNZi^6TfGYbIl^He6slX4tqbj z)REn)2}T4?mt~)K=rN%3zaCb{!t!dUtxfQ5=t>qlR5h4oio{>aa$twjpa4?t8=1Rn024)yJ2?g? z{Cj@m$Vz_nf4cGBdhnxy(B(zTy{^V809qGIe(4rS?GEQN1<|A(NSa4K25%6J=y~N# zk3=LkQWU82_jhWNGcD%HAUK}vt-X8s&d+jWUPZ>Ydo_2f-P_iEyR2f!0|B^Yozp6@ zm1w-40}|jQ9!vj{t$J5jO}TpSAf)ph^zp{35RzdU2*NHz{e~1$#Mz>5m)6d~wo(SP z%uT*;W`k<4Zj0SgU2$M2MMZwg6c)WecLxs`IL4fJ$%$knFiCELWGLgCggf1)Y;8Do zXr7jO`hRrU*`uVY5A2w<9PpB#T+%8!90aC)wU2P+2-iFZImQr@aW6L8xLjOSfSwKV zyxbYy&-K~Y+TP)H%2+)D8XqB#m_fu!SfTg3gOWa*5_i{g_AO75fnitAtK*zeyeFYU z?5W&WS^nNfD)gz^z}y>HRg)9c%bC_RjauBqOe=<~nd zUZ)xNU1dtRloSuoj1Pche84HPOB-t#lAsM54<~pQW5zDN@H}@}^75-`J@aULX2s%1 z&+<&K)(}+8S5!Ef;1Nqy)g<3mJ)?6h`BlVKb%*?RlMgjTcyi4Pof-Hqza0@qY@~^4 zxhu#7BtN@)>wu_@hvRUk<#xZRVDz1>W?j&Rk3vM88m;cj+B3%2;HYL-qv*zOi4iJN z@$P&l$Nljv&_(($u^{A_^(`Z;6ulhmyofhhfIsgG=#>obS9CfNQ5My6enf1T@0aTQ z6{@#Ml01~2?=86P6k;IfoZ48@>==fSTPCv``fBG^?bc*mo&QLx$bLiAxHD#H8giPD zw8b2<(ai2c*z6&v7 zy+@C4Nsdg&_7mB~YR-v++F6#MUd9WA)G2$1eq18Nt|<0I6n?u&nwJfdrkg(7oqh${ zgb(?c9#P)sNce@c)h0$YPILrOXv)&cF>}+@QOB&J2?+==*g<1r2)2n{TK6h4@b3?$ z3fz@6*41s}Ik)KQV{gmxy=BvrNzWrYZ}e}br&wiRV3;P{_H3D*54Wssrr+-!wOG4| zCC|K(5eDD+hra-amY0^zlNziI64dVM=x2OXcDAG}?{IV(~1lw|MFA3qeb z-ELAzlN49RnlITE!E`wycH*#3ZzI=eWPFo3CMB_~=+nW3G}s|DQZPDH5OZL~R+mvd zZuAX1^OA~SxR()c~-&W#H z6=ie-9n-fRNkadGWzKLvOutwh(<2VlKONjA^nip*jn{DpFc#2qB}FoGr{#w zrs(!sti_vaERO8-hB0BZRpKKAOpb9#_BjepCdTEvq7&E3aPmj_!doot$oPu`8VMQ( zG9d)fK#Yn?&>gMJYE{48cdDAsrnw}lU&Zcd93-a^fZo&C6gEcQT1Su6#e~UdkfdZ@ zh|k`N8^O1~O~QQgRvHiJWbK!b;1S5H=aya1tNTvzjHu(jZE8o7X$#C-5Kpi=K}d|z z7`c|;3PwYjf5L?R!@m~+Jt;BsBhqs|fu$Oi{rj?dPQTOVW;OKxqMp=UbP*A(hj%QdyF+rTHH*56JH8CWw3r!-xhWj1Ya@ zM(Dcdi6-a!Tk8gbdT}XxH{Ff~2bLIH>-tzf|#9?^{6HtA1y_?D!8^T+PYGMO4J{LYhVV4$2zuIhK z1BXA*F6u0t1VS?#wivEMp?$?=^{>w0OCvyQ_@I;Y$*&39hblT@RaIQolONY?JaV;b zR6x?}Ly=`h8n72T&&-NIAh-Lmu`wWHg>^TQx8EFzmkq3y(yvi}WjVR@j)7u%yX;bK zbzIHu6cxzX4m1vrzz#p1sX8THxSpUR~@jcxmm5_e=#>WaR#&hBuCrGPK^d^~FtL$5U}yT#`$o3(Q*> zP;ASl)h^gfiLY+ik&4Y_-figuPU@gyykJm6Nm|ir?%912fUk-u4A0dFYl&EC3GkLq z?(~r*MdyQ!gQ97w$t~W{RyQ^@G?bSEU3YThp-)ULe)nkttmZk(dFj4SI4)1O$1OOc z&1l4SL^-6Lc}b-6L)vbB2vVJ z89>9PRqk|Mg_f2UnBw7<3E&ICz=}pK8CKT%nK=z7%)NE z8F9JViS4m{7jtpLF<2L73{0!)Tt=kGFJiU~Y@EPbzke@rJ1a2QZr)%5_h?_&5L>-> z5BLL|HH5}!jbA|<4pOEK1-(UbTfI|Dt#XHzNyDikMkRTiIo@P_m9ue{oqs!Upyj`Z z^us{r;0ma))Zz06h{nT4xGln^%}cNFT7H_oort#~RraprL}osuWA$@Ue15SXHfq9V zehEaIH&o>x@>Ttfqbr(T(tr0=yENlIYnGIz^A+{}oG$llR@-6m(p@kvM^YI_ z$LkCjdmefByt$*SMZt3*Ri=^n%0YED(3KCCwAMV8QG~#lVSrBdR5bYVq7j7WOwnMk z)`cc1|A^y=;ff6V7&s(V1s2xgM?kplxIg+z(FtyL3e{v0}U_+b+QT=Xe8;Yk1^rhmSmWQ{LBR@{9u=(HS2F#`FM1`niN8od0xUmTCEvDd>>SSGJ3vHQHBE0t^MZ` z+Z~@@yXNbBfvqYNaL#;+UReG`wrm*~CEOXBK;I!TaFZ@5&5qRi0ip6mEk!-ue_Tjh zvr1c;lV?>5`ZntrkvGza&K1**NQY`d`Qzf>#{xCX`0VB*vCIO3ilI?j(*PG$foB7S zAYQ50l=)843(w7dPPBX?Dgf~&`ZwWZ*vka?GRG=dxJ0#NO*AI~W>l>+8*UeiZv>Jy?|kVT?s-e+XzQcwcS zZwW=>@kp$2s@CRQeB$VK#Z}*H%S+K%r7fNgjP5WMtV+o&Q0vD8B%p0o1r!B!S7K^z z0)$gqN_Hau?qY@kbz3jj8To}tlw(6yFQXt48-A((j$FgL%t>%yrT%H@I`oO$?G6#)3mDc`ERz6oBXEWQ=)JYZ z`6L6Um8&tFjP&hxQQREeYWcJ$Ju0g;D!ZGW3O81+uTYNF%lS)o@W=oa|39=&Kr3fV z_sGxoE?{VQ$>>Z9Xw3t4>PDf%Y#w2f$ZqyB`J;1QAcC6pb96jj>m{Hikh`OM>+mqp zwq7WNU9?F2)7{0Tinr0-12~jP3!H$CDm)SNyVGeoAXWCP#TYML2Y^g6InUQc8|$$z z>ZufXM3#%W=T%qS;PW9gY;5}ZkL{Y5cQ*&?!ogUW2{ zPJ=5M2%_Xrmgswu0C`A3wZe`}d+*40O=yc|lUsJ#u)9^$lrD2ORZDF@cbP zQiA=lMh?&!f1nCZ89-ZUuEn-OfApejLt*yQ0eYZLElT+f$*$cq;gZVpNihWIQ~bvW z5M|m{*Yu_$nuE~%4zRJ$do1U|7C+NO{&_=NRBYgcX<&9ggJ_rnsxPW&=;b=* z4+}bVER&@#MsHlay#+}ls5gK7RqQ>VNB37myI21rJ&p{RS470^NVA|Hr66A2pU4ym zhUE`B>L)N5RVoE0V3fSZ-U4lhe+U<5Km{sa&5ez)VhbO#oq{p{d}KRBbD&menP?F0 z?tz+5iv!HtnIO+n4gh5uc=zF-i%K?Ud2Q`iaZ01-SzoMvBIO&>TF)~ec`n=wPrZgT z!b-Da@e)E>XJ4-4AhO3`Dq*$K1ELClD7w44CGu7Bc{EUN^xQlUcSNsA4_En<}7#T49EMPkAmH#|}sYF2@f&cbS=n z9Z;-vxLvQnY|kcDfG%6#(wu>z{WkmXGIviXy%S;{xKYH;j-XxPv!5Z+I3P;S9sOy`r0_&~pyNv%%;+^V{< zIUET@M_|kpX7+c@v(K2xayrl&Jx*wv%dn(gqvP4w+Y{4{mZB=4QdI67nNGeZ3EbI;kuEP$oS63%Ak+zSMcMA)6 zsPqzj&i4}2bsYY5MF8bL%W0%Mq$gFm6U3HgTBh%Z9kOouWb=+OROkU%{ny>Y{}y!* zO%x*(r=XH2rIM`$3bTM$e*^ZP81zt5E&~Edz&(>iEW=>1X?;JYyNZSe+4aZyDp=Gj zKWbOLL`DL~7=IBFJw#UNp+P4_KtsIM;S(^#1I&UkLlUUS5`pBQ18p8S#u}=*BPkt6 ziI|hIi3|LH3$B(=wF>wgJw5AdYJ$(vV`UK8P>%x1fo5-^VDQ5O;13yb(h+O0IhR;N zgft*NQCi+WD97kMewhKvo{V!Hct zTKfNbT&o$-_^SZnTcF@KBx1Uc?*PY@Z=o?qQ=klg$HS$a@aoUe<<{0`?Lah;G+L(N zRG<|C9jcI}zYVJ;x8}DWfRg-xOX6RrK!-Z;vTa{cc$a%&{)q=qhVuuxJ!~bcC${(J z=hZE}F9Kczwghkc{(H3wIxds8Nf6TXxRm!GXTL#~`-IdDLzxClk}boBk#6jeBpj?g%YWLlB4W6^*(=SMdpe5~$_9`y=e21q^d zU)+Q>;ghgs^Fj$LVJi#7`ZXzCj;e1%w7TIBBD*y* z_3w}VNWd(Z*^Ud)jI@45nh*vNl@qf~h9j;Ni3=7<>0iVpiY(IPy~-MMIe(8LD4N9S z|FFzvr>d^*Z7AaZ3qAZjOwIv*O}->C3$Ui3;^6vwx62OdqyNYpK6Chw_)6cfMtK5H z)v`oDM1_BcAg=xwI_XKPHFEHC{8z9ZaLIs;G+=^JM_2cmf)Ek~)gT#FFBk=cAS1@D z%+f2IC5@8!#JFkUPpj%6bX2nc+Z2W;tR0-H(=996;|N(Q~Wbmt_!J?y|g9=oMI zxBvzrT0(=a9U}aWT}4J>PHM)~l@n+GVkPmo`k>|FTnQ%v&=wV0>cp$nQ69y}7Xu8- z2zf1c)%5}6MZv&qX=spBfyI{Fe^jTaEO%_}x7yG@Ia&Sfxhr-Ud!(E!X+r)0QoOYc zCs;;o7dOfuDLN4`hl(u6Un@un=-gx)Hp&A%)30uCDj$NVu+rN9~*$oXIJ3Px-N z%_!UXj$nq5|JQD>*v6w}cb~uq1F9Y=)zeWwM}ESx{2Gng3PBzBWQC?fnN9qELp@gcu_BL3vGwV-aNbpRZbh|GJ~F zT`KHicxdSS{2ZW3CjlW8P|sm9(EvHX%ItTlnwyz#nh_`dPiU~k|8Ho3bls=_%;yeP zchE)lLga9|<&Llh3z$s>wDq(O5x_X0{x#34qzsN~$om(^hGC?9O;c@)MGU z7@!Bw|Jf0M$S2avaU?d#+4SgDIDoF?6%Z3*WuWE&I`{vyaV_vrZC!ky88aTCNh(H| zr1A_mLf2zF$}`VPUBVe2#l7o~~*P|iw8ltNQxk>Vfk>t_1SAA59B9h*9Q+X8M zIy3oh_xnEM_v3eFpMBQZYwiDDYwiEq?^hF)hyJY+2|0wvo1Y6C!@|$iW$8Om&Qg-S z#Q}F64O!+KLWyZ#*8g&o351<> z9svn$NYyux0+R_CJZ$(e8nD69OqyBZ#;963J4@}-_)kivoFBaf+Svhu@HvZwxOY*t z6fX~!q|HtOY5gStyOR%Tdp@^PXI-@vm>POj-Arwx^X}eV*mD~rUXr_Mc7g#HS-2_? z3TWeEP&_;}HPyF(;9z&Z{PHDNg3@)$4HzQ-tF7M&R4G;hlRDsj@#Dt`yDBOu?4Mwy z%65?a4ZHNTiAZv20%fT3?7h7v%S*((AZPyFTwp1Q_sKK|;v%vY#wuJ{$zKHMEP&5f zF)np=`qS+XzcQWwcx>;terwQ{k(fiC(l)}EOL!&v=uPH;2{W16=D$>^X^1W)xG_SP z&Hrt}cbbM2k-qVjj8*#P^<}08LieYap)kGxe9LoJWL~89{xbjPE5@CP{ho9-#2f=n zWRe6UP%7-sbB$ZY%18C5kqsE8pE@m~rlNXiXz2D1)yR$S1qj+Z#hCt3`6h;zmH3~v zuHn?@!8}85%j~lNMiO5|9V~RKsJMV0+R@FHT+;W_Ik6sn8VF#f19(`MjxMu1C+U`?aGOgK8Mkm%_1>da)YC8 zp%X@43&v%MfV@O^Q{A^c2$X-58dMo4r!8N7qU$o-z*!{H_F&vX$h{L)h)$q-N-Y=Gw9f>5v zc0Tw+^`8LiAX@kbc$p}apx@LKB6#g*LxQ}G#4nW_8AS3p8cBAB16aH(Gj!SY?SJOm zjdnquf3UT9p={e=QDs(#KPpovLZxau4f^Wm<>QU3XPD2Ph5kOd6wr1jhm>1n(v5|E zJeiqk?UWZ06ikv(84}xzvt+xJ;beucJ)!2FJFs@^ZgrI8HtP$!OwC)DM>?cQs5-Z( zz6AT(IDDBnu#UWzO7I3r65H&BA}K40t^Ac`qU3P5GW0=ZGOk9s{{w6=eGec2Rj1CA zqaV$Tbmp`xV>dr`W1XvoGa za{j+{X^ECjVk&=l6HFH&sejV$)C1O=q(cuUM1kUvAgvoMn5z+aWsyJKgf7-FOLHhLd38a z+W-A2ZcFsE8bxvg;YJ&+6+a86MHYRLp*oz&#Zs5KyGx!+?M`s&_!ScL*6I|QDz`p% zYilc1n%k#;3yrSLePaID-xLlhn;l*=2Xc{g5}9>JVno&m|LQNB+E-b$Ieq^L_7Z_b z7ZP#!e?=(A*XT}Mk*+zG0*QZTFGa#WcqYYh55h!DYqFsdrHBLhBcjq~_9MCHX6)NX zrsiyJ)C4m`oQ~v8G$)rzap9nmt~g?G*qa0846onZ8DA0lnQUqts~KaEarSujXL&E< z{dzGyxg*#5?a6M>%R{oOE>er6M|oIp3mcBU1NP3x%cR8-^>eaB-`B|z&to34x6m_I zJ0k{J)Qp)6<>fC<_BNWkTP#<9dQS68O=U|xf8c{uo#wlqvRLki57qYtUMl;?IS`b1 zPDx5BR17o(IW;Rk%)MND-rG&9#&88evv)h@vRYMv^=zPY$cM?FfBNQ1YF;0|f93A* z2HOPf=DrLPlKTNNMJp-d=!dK)IlaYVB2J2JW&pS`5tuI zc}^*2&Su>5kqiT8kL8rG&yoU6`--j5Gw#c9;BYju?9{o(XnZTWt}9u^)^y*eom;mi z^037b98ywLmRulXiw_`-Him0e)I7OdB?K?P497yL%9*rb~*FCX@ z6*NC&*RGO$f513xGLZV1blayZWN0i;A6^PQ)q96P`PuddiS{m#-l*yho?is^L~zuu z?b)X0L9-itkcK1o63je+X(L7Z0f)L7)Y%#=pK@qI*Fv3_37!mEh~><*EIqnfm4T;G z9ou-oja;q~fR|z%5*@A2*L707-`B@C`VD|YAp7Vd`JGvpZS&1zoi<{%OuaCjX>(Tt zvGs-68yay-RlzFxP;UV+6Xas6>~@P!wRodp6C4azU>4)`rPYpnP!dYV(2l=Km#}BM zwPl?lZ7%K@YiXgFW|rtPPF7v+K>HxK2kjzxTZx+Mo$_ES+{S>$ue>!Us$0bn$RK`3ZxME?0V`JTkk~AGW*n-Sby2FEng7{)4omfi= zBWc@SPEPa3nkadiBrkgEfUmnBGdvC$#{UYvx=E zBOcC`$d-i-ER`z0hiqztL5}PiS6=_Mq5+x?B22Q(JgH6So5531mediWjQt|22_v<5 zfd#{1k4oTk7G{m{Tw9^!bSNnomI)4o{RmY&qBp#8G!fX|)~Kl75>xKBV#q0}Q?Vs>_2=}p5AtIlrjz|v~SDpzUhAch*{=t=7E?crfW z5+$5%P!(e6r}=x-9|>^b{|Hf0$fNq2#zq}i>dmM>tmd#SnLQ2UVQJ=u@)kJM&Qo_evWDCM2lFSmYD$MD^6*3))eZku72uC7uxn%5>#YeZz)lxtE8QmaeAA( zJ+E+JU%zP@vQI>QbGj_?Kj`dabY(_^%C-OeN*{k68RUor(Qfe9*S)N8} z))&@6Ddfkc$r84bLSB_ErmgZjHRLA4LKJ-MhIp>x$2)z~8ioz+qRV}am}hCMPmF*` z68f4(4L=eGX9(rHB1p#|-N$F;mQ9Wv9Da6mXxH7v`_&7(k4ILa?UOv<8@b2l(zwkS zjfA)}Luul%r*FWPGfYt}Y4_LSwdpeU{o{0Ht|LOlCDK3b^0pmU5PThesQi=leki8l zdq}e$S|Y-PY$Yr6$xu_^LeJXo9n zQlLR;I@QsZo(BKL5NdhQdVw_04|iS@*}tm^hvyhb-0@PISlb=#n^oH3P|o0 z3nk?hjJmj0wiL>OlmXC_}J4Z@ew#YAKjCwwOAXZn*6@5?X>B8nJcSX5Jb^(_931Tj9(bUMsl z9X|9(uZbTiZiw)v4s4yVL6oSDs1OqVrisi=^6%*%LNH#)tF zZj5)qJ;boI!9s8N;Xvg}x$7A7&TeKv5R;q4Cmv0#yB?l>^v5uLGd35Fu zxQ`eLrdFKbGT(Vl(%6ih)VJT%Se|V*lZz+uduCELy_-8e^NJ4g6JnqUv4d_3HV(lO(@Iil zAxZ(yhiyE^&tF@_WHl+zC_@7<9TZ)xZZvNRS6%i(Ss5ir&H9Wb!;s(8T+lO^a^c;E zeN6>kIJ6?3ms)nFqOMLM$vw$A@!e^;pbJ}^y}Xbge*?d+_l(1k!({7*@hwe6zn~_> zH8@h({Y)!E?)rko>v5|uq`lXiuY6qa{#iYo4(y?rTyZ$GmXkgWtp&11{q=6p>NR>? z8}YJLIW|1DN2hZIS@dS?rLv1j=G))@24&^D`(ba!XeHzX5MNd!7IQ literal 0 HcmV?d00001 diff --git a/doc/images/unet.jpg b/doc/images/unet.jpg new file mode 100644 index 0000000000000000000000000000000000000000..49cce6ff8da41fa615a18f74e3fae1c818493c17 GIT binary patch literal 41960 zcmeFYc{rQt`!}3UryFfe7u4RV+P7Bi>x>o^6%{1JHi}w!?@7rtUyUq9a{(i^%Jm2Fz-sAn_x%0<;UH7$}*LE(S^UTT6$ydO4 zKs^IJz?m~=06(4n04EH5hMmRf%Y z_~y*nGv9uD{tDaKbLY>Uxdb@#&Dn3SojZU1`}-Ge{AhCXjy>W<5+|4Z0|iB}shO|e z+cIG#pX99aisrtFyE-4Zm7o2xD)R7Ub_+$++`%#BqL|7f-N>BW%C*zp`A$duW9Yw( z{YUTT&R;k!RJwLb*f)T4XU?BJf8lRcP7AMndpfZEkLPYEn7k;v`2cap)c$SODkqmP z_(R`>4&{UnxP11s-L-p+7s`;+gt%`RD&SUTCMA#{2yX zovFsZpBA+Q@|?Zh93=l|t+UK-*`LDB^2h^jg?f#2j*D@4u_t~XltCM%QMV)&Tx{Dkon(oX1+Bi;A(YfN5N@`)$a6D0@9 zIEfC;Lo-CzrWZGn!-70m>23t1SmyW|898I@JnT>!c&0uVLvH5;fIq5gU;9~L0`a&b ziUs{GYuVY5_j!*Uu$dBN1l;Gif!DJtd)Y1~t)=xS6MgGFH}x`8A=qm*b(`96OqNJUckcM*SO>H%ej*gZhn#(>l&TA@4#iko7SL|Ff zO>y-qjHGuNs*GA5ykESX3r{q-xvdPVD6gXynCBn_W^EPC0wCK$vC$ac8Jq7FRrCy) zikKJMY=Q%uc<1suBSQnCvsy3~*htz^`H^m&Cv82y9xuK+Gue#s(zD|?F)J;*J zvv70V+-5l^fFLj#GN!s)Ab)5egPG3+4wMMH7Qp%`i#=9lDYG=w9`EE-F241mpdaP| zdt(zBF4+t}g1zSNG{1u`S6Clqzx&{|nQ_rbD(j|qHM#3o?v~&Dm;t^{6>5IVo|1jZ zbuE4~lk^cZV#1ypruoo@RDsJHh$nS_6xXXP)^KXLOU48s7+qS)i}>Ddxbkvl$8A~0 zH88SDOqT%hA%G!%I0++W)ns*5$7DlkxBgF?N@LpZ`;Pjd#kH`ijDd%`eU)@N2CEA) zCzzWg__p&4f>*vaeW&)FmV@XZ=gYav+H%5iaIYu987LBWD$JbEfKKmU*ifmO8LXYk z;P*+$A;CT?TuE!}$0x3CVzM*$aGg?AG0u6Fo{lBQt+Dv3mu4jb2VJj`m7-KnP==G7 ztkb*O>i(|@6k4T~rr!7%%T=$yl8M3uQ77h9Ju^L*f+1?6kOys8cLO2#<@RyVs6cHGSOre1La!V-Fs9Td)BY_;@JY|Om;Oj z0du%*y-RZ?>pD7KT6C(8$$X<$DvTrscR6o%lK10XC}cc+k9@kf&I%>ej{-r3S29X$ ze_^Vj*{+8N<{JtH3iwTGDlI5ig_z)VXzP=40tKE53o3a#%y#xKc2&x`VGlf<)q@ol z=wr?5lq_fh)IAQZ;mryMQ89iid&7B0ZzHb%5MbUb*4MqAoF{dK&xGKRXKB{INJ);j z)X8J2)Hhz5s5-i}o>3=EImT9|p)}Q4l;AL?r6&bUpy_x`uh`o+&NOKL?8h6QZ-^9= z_BPsZmY+KTxG9k>=!`@O<16-w#n}8ojI5{k*Bqk``+CuF43$#* zh^k0@*F=*RhJ>F5BBiW4L`+m#yZ9gU@(am^b@Ll~Z1*iI4OqBkoKg2WA16EjEZ#_t z8>!Zi_HDndZFKhC(h1;+ZI3%G&Han}m({eGG{PXkvvHdY{idC41;6r|D#Kc^>|7}! z4v6bzzfw`gOzfO-J)aOyXt(HFTA47S#&_QuF!7RA@6d3j=)#K!A3FG%=bMpGv@LTZ z)3_a#`n4V*Z?1c~ODvMW?_WH&R7_B}MyRF>jIP9FO`Ly;fqyCk9`L&j1F5j=TGL;T zGOsN=CpI~ykr~MSlaFBrc24i8xmUN)w%ZU1L4|cFW`xw02%>icbWQ-KB+8QH*f=zb z-R=GuW@a_%1YoCsPw7La=f0=8r<-Y?0(<(x(-S~F*68yIpfg8S&_DAHdKi>i*`3t@ z;jY?J=o+phPzyEZx*J=iWwrF)aIZd-3zpb<_aBH=z}aNKqcuU)_Pg-;XBEQ>>Lz`P z3F%fyAPyrSL0UmxUvt_yZtGkz9=|qk=Za>b8=~zFujL%23jNGa%^J-v%yka%ZSz=(x`W#dpSY-r_ zlh&4Bb_zZLSmsn}%FV&B=TkgorJY39oqlr)z9m5RPz;(&|2}`bFR&08SxX+3HC$7L zK7xj*;=a@lAPW!Ff>FR8h5bF^oUU|LMMu@RkXXvx(OnpDBWDzbRT#<94-;u^pk?m! zuVjb49VQaKc#05)rD7SuEcvi73(dCV*pH6rVCK-dDOV-518MmiR@H;o2-`%ZvZd`$%ep87 zmU&e!%SBAkG%$DL1n>a2Q+it|DXMM2tp|=5Gh2)$cL7uF4>KhLxQ#GHA}~>%SI?m zPY5YG(j?Y8V0KRoHG&&B`s(QsCphRw&3B`GCN9mYe%P?qcpOo;hu!1#5FtCHdT0kr|Cgg0$s@5&n6jZ-c#iq=lpLM% zRF%N;QTG#oY?%1fj^CG#UiHlTc~tIyhi5|o!G>48NaV4e5US^Ay7?DKolc6xm{otA zTaVP$+`DV-4S9n%@+kz~$}BJ@B};{%XP9jKR%br*DbBSNxx8Q`Nk-a(Fo@pkV-e+Y zzw#a3avM;@z9=n3C`(qR9RGP?Vk-DPL39wa z&~R<2zkF`JV<2IvuoH<+Gn(4|DQn5_1hBnT7*@QqlY7CL+~>zIgemn)!>9CKg7JdH zK%$R(7jwopBa{0Cz&_vPacm&)lo_UZ?C$zU`iylWBet#^{bl>%^K=w51ZE~HsmmgX z-K?z~mmQs*TTxSbzr>l>ZXvTw-;cLAhBd0l?K+G!2@Uzi=M23b7_zzHy6Mux?Z2(y z?8|F>CI7>->~@b3#Bg*AhbBi3yc$p1*JFOpCK;OL*lN_a4lC4Uda7zI{Vw}j@Y%0cfj+ z(|QB~ya;}1ihq`=UQHB_$>%vp3X_w~j~)keRZ0&G50dXb?y1!ct*W*4ax9M6esKk- zSSp~{PwHW!f{T-uY|O1yg755vK>{+@H8L^Vv(Nm|&dd33^lki=A8TVG-2!!i716ax zT^I=W1xEKCXsT>~k}mVMv|pKAV3wN$-5k7-3GB_iDkSiYx-s=?PM#S-99)<`0xUGf z2&)>DsE@Xf1RTI`JgKb6ao1_4D~L}t3&7lDd_qG)>wDzN?N;_r0L$pio`9cJBA!5z zBzP62k_~w`U+s>D<0ZLUbObtdtxjc`qi`B(ZkAaT^hy?6jn9 zNK=clmh_2d>oi*Z=s0--@MD#pp1mnGA8=e!3T)Z<%{4zII|I_7ZJC!dD1PxJUV^%2 zmK+4fwP@K%wATh%G0gF@2Z{&l>go9@Qx_C)vJ(DHKehx194=AAKQ+C+t=8DNl$u^1 z>D!rRpraeHsC4?1^8(uzB9IywB4amA^5^qA!L3R88~(!lvp z{m{gGPSB7JU-E5ry8#an17&ERKUd+o6`XdybpcvM86mP+rKANSQqp4FJ1a7O>{h!| zH;ffBE+M$lqp&g{BJ?7XV%0yNTV1pcC!1 zgM9Mn>G$Y{)uLib!gMB|<{#cE5?nNBk916$p&^Felb8MbJYMyAv>svMR>p;@Ch0kQ zx(&RqpUVV;ZwH_6FN9LItS)tTVjEyc*StM1f<*9xZ;wQBj*>lmt^HC@0LKNVeBb$p zVNIduoPl#Rw_&L3z+vtTvkrqu4qI6{o4DYYF|tWt^0LM|=;XY76^QhsCflN0jvZoKh2Np!D+|>rwK1-r1Pihqgh9niBVN6q`En7)chst(wEVsspvXcDXCBIPq}8F z2c*YkQ(9Uvmd$Uz)oCj>*k?H<)7@En{?!S2C^uz6!V7fE+%rncI$VmWhY2lT1%R|z{k1FO>NiV>R7=YCxaL^ZTh zWkIiyq>0h-NRo3@JobuOTNDG2quv*V48Vy-MZ=VQ_l4 z)UPVa>vjrFDDxrCd4ohRT{>*cv`})bkRNz2E%{>@%~m~O_g>u_Ax`mT z5)ZiWoC-)*K6aWqlb{;ZH%9dcuYsBl#iAf_>D`xr_mdJklF+et>-W|LFz@-_F4{N*ETAN6BUDibmz6aM&})X>YfB?QDg z%)?6iT=d$!A8=V#L-hqu1^roH zm49=y$_ao&=bonqrmXd7PlYzTX;-PtXGehc740+lI)f!V>7sx!zuKHfkOojlXQ^{FIIw%!!9jO$s#bYKzt}JZ~OJQXdjaufr zZn=;2M|oCp1Q~U-zw)Ws5wLCp+`NibayO&HGb_jxnff~cm z?Bz7Y&F&=1Qi+5(G|L@?wQ&C9KBIcS_}V~ojzXNa=@}Je{sp9ZT;rT%*)P#TyjKiL zIJX_pSpskD&oJR?$@mMft62S;%*(UL?V|0kiF8!H@pDDCU|rd&eH=I2y)l!?U1zyV z_DVLjEu>2^VGi8HouSRQpJty1#w&O*C$}bJ& z9;D;+n^;(7g`wOfqrko+0%!PZ1)~V($}Wb0TQ`G(HsRH=d&Yk z%oA6i$ewtlriHzV@(-u+!im|1dYQ2xQD7E;=Hb0cZ+Nk{$+eGIHwn(>v%#=vw0P=lST5pXr=za59SQ{qag3*`uF$;7*+%FYm z-E%JUS-HEiE6@T=(w<{1`Rz?bU9^0@+HAm(PsQ zUtf&p-K4RU?XC>gwV)B*#lf0%Vk~W8e8~oBJBB}^rkIgKa6I+~vOUy_iGcITGe1w4 zSj^k*OQK{%$A(GKCR^@;w(=2W-0;^_#RfUu!YjFrQ1pO%ZLynCzP`VNevF++qYdu# z;B>1!o}&~~Mq40#1v;rY-{z=*tP1i}?Tf)DfZ}gO*n7~zKU(Syd&k5OwQ%(g8OPgv z>ub)>n|r+PRlJc95L`;dSA1!zrajD}UkS@@Qq?cK454{swTRmCM|7=Cc8l-bWZz^wSL3s^V+>=8Xc z7Go)_c-Up^l5Logn^$<0WBG*K)|0ZIRodP;Q=YeeaNUM%Zn57|9@y%GmW8GZdtN1biM1h8?yQ{vq`X!>4BmV7osdFuyb3{s)S@{%^q-uw^!-wKV{rouUmz9B~1>nNHJ1X5Br z(?*ReOH)8eS~s%8%C%!2bhr`&t~Ora^qGyZBHNX3TtFo+ZQxcA8-lOiO#cu4{)eCK z*Bz1ZuocsK9E@yvOI$@}#B(+zdg)Q@0@M|u{;gVC2u{vX9cWt`#wujq} zBHY6|KW|o|nH`C+OtM?hGaqn*J3&UF7+Go(XvXC8^#otkhgC7DFHzm6N`b(fVVYDh zDaf^!6ct}7f#jAC^C57n8p_DRXWzQf;A)RPKZ5^6UEDQ_XRGTB*`)fcbn3q<^Myq5 z-pgugfMvh3kHazIR`r&7pXPuR=8JTsE(Hj;KgEHFC{Pt&=cR6GQmb^aRE%T_CygW- z?3WL&U10s5J1a>^*i?3k^Zv?k7y44(3`BrqJ3~z@hYLR~`tK<1C{;RK)VpF|qbk9y zC%3EGRaWRp-Z!|LG>x`bDxx99V|8=9_W|Giw>$Zs2VkQ1co+0~`_r*_gR@4AoPp4# zKCNM)R-Wgp=!gm}(I&g{HT|)i8U3u;zrAu*0B3IhYny)R`1$-3pZjmLUc~)JaNIw8 z(xZnW!?ygIeoGQ2YN<8u*V=u4atACBW#}5AS{b3eafdNvR+YYyu78T-$Sbr=_sYc# zCGw5GfbQ3XU*6V_UwL16WZiV$Nnj#(+q{Z>)b_}RqgnSO-h;cgVZT-=GWKk|oL*vVB%8p#NT+qvpsp2`h~KgfaWsY);0o?3 z_k4f@8%*F6u3Pi6!g6I>%zx=Cutwon!>T*^3Xd+{nh{{f=`2@6M3&grVIx3|ST2{r$iZLx>wCW+F|}rL z&PysPbP1ew-K=)$TMe?V(QCM9KvHo~@77okYcz#E9&XUKAQlevy8@^Dw~M~#G0JaA zk&vEvI%+aj7dM$yVNhQ)V+*R-dMt-+r;+wJW1b#IeIwtt`j`%UGj;JX6J^2gamT z0m1esQLbjb1kji{R!yoA+|E6%PzgiBjIQ8vfbqOn>2YG9v9a6&9n6jhmUh@BRhF9m z8B~c2RB2RgfNw81gbwZ?zNLd>-7CZ?)U5*E6jqjwJeXV}$q~XHRD`e&Ld;G(MLZfN zUYn=!MwY3keF3A|t#=AKO+xxY12;l7G9FGjRtm^xM~@bW+Z}RYQJcko6I=}3P>#@r8tY{7xWQQGH!G-GgvLpl<%dK@#r=# zIZ!o`b9~6R+m=hYcBV|=p>chGf&NV*UJrw3 zGY%Np(PbV}S{-ohz#%7>xi-{e`b?!mM|bSvPD9RboZ>aj5zXJP zqXn&>bk^UlP`z7UG;8}|Uh1bX6gXS}G#<_^S?Cj`%W@EGZNDTZ(L`(+PVZt@buDdZ zgv5{@TSHN5_qz;}CYhKLvFn=Ci@{c_$aKHTMc*3~Ve;THl%1%9!o^z^yz|Ci(9plq zad(Ux{emZdv4>Aa6V`bjJ?#LwUpRzIxFW_}k>a(%syCocO)YU#loXKU42)Mx@Xj63 zAP)9ks`7wPik8%r2(X?=S6VFzr$%{Ydsvzt8cIZAQfX7uG?gcQm0)Vk%&*71nFaGY zGY{~_@(G9888;-b!WtuU5gv(?&rbk46@6T_7t$TnxFzza_1NM{D zQRWM2I@GC~WbIa+;uC^NK;0DN?{R^R{OP4yd4B!y@SBe)UB^E zJ)e7rufXV0hfHu3=I-X6*)PCK({{^Q1N$&$&`j-Xca{`7ZR@vR6Hj56y#11d$xF`n ztr4wX_mMfbRt$3MHr{8?tDR3S?1g6JaKijbSX|Qu3i-sHDRKMHCyiYd;xuTz{(QdW{3qf@R|jH4QeP_R0qt1W0ULumA+ozZZ&)`sKCO0h%_?xVoR zdBbqKw+J|KV7S$}=aJ*P__>U$s4Ai-R*^cpCjoLV zeKAG(_Fh+_Mu4nzp{)g!wkdPHXh3R~FR-8J3Mq`Xo77cxLUej*ju`V5(8y|BKNscq zpf0*qLd})y=c{X+oIe?>m9^-D>aYvpwoUEsodxNxi1rK9_XqW_2PJTO0^^x0By4~Q zCdo{0`S4Dvmj5N)65!~;ucN@8nf6D03PWu-P|KF;znK?I@K(5o_=9JoQ_C2iKt{o$ z7$JrB<$&cwvUXR+?>hC*E4VUqy9$yGHfBt#-1QqNq}e98lLH(aJC>z0X*wQckt)*O zk<4(Dky%m7d_=Cxp91DewHcU=jPIxc9{j@fkU^uQpS;X6K0^q*Jn@{R>t|uVCYdpCE&B2!WTrr` z@znePBL30TZxeg(wljRpw?SiE%JiO0-3}OIL>G(~9h*cL;WC>0_#IvZ_e}XrcPEaH zx!-6Urzw2`RQ#!2oV|ScB@Egx9KdI32CS#L3F@|Hn9!t0@~}1o-;Ql&_uHyYY`>_# z5@JPHPCqhD-Mok6(QPy7mBx>SfZQ$so)skQt+B>t(&Jbp{d6tD}kk(&{{CULbHS2>QMYiI7gg)d23 z8SQoF?41DK@!GFDqgy@xsmg>1cW!(|R zs+iZb?RGUb*uFnZ@yG1f`yUo zhPl&G)o`9!{0{Ck7^uOowvbe4+U6x6IxcAY(&OTXpkp%MqY$z9GCPHkEe) zlRmT2t2%Sgn;w=%g}cB^a(>LW>_@uHy^Ot6ZL|}4|@UzB)r%-B^ z7jFT_{UAh2D8)!|=QpZSfDuK&7(j{=thyGt$`~{0d63~1JJr2Co&2p&+fL5dI6B$4 zBSKQzp`} z8X6`#fkU36Q?^!gS^+BLohLQgePZ`U{v#_9d~Q!a*3M}~%3WERvr@%v6lt`8M4fuS z2inV2R+-Zsj#R(8F3lu=Zca4@3WTs#PqOS;G7aqE;<7`iHEyU#+?o_2?beOm5wlS| z;?xv+=GO`FM|$}!*zy3`!%J3PzsGVG5as-?StGsKI`geCgyEIfWr(7!B96TqD)#8WG2*6kn)WlN;ichslkPVT!@FALf)n5Isx`i_FQ(v)pW-Cn;$`M0 zbF-qmFtF!aLZ^j-uhL$}8@{R{$#h%x|4vlWeq2^nn+5k$F|Cb>_ka-IiD0B_oJrf; z>B!9x$pnh$FW`!u*Sgi!(hfXE-t7)wA6HztYVm~Gq3NGLyCp|VW zL-<6dxyxRktWQJ>vftgBd4accEi0`iMG|x3hvc{ey5kh*?wTasd3#0fqf68S4sii_LcJ2T?Ivpx!iG?9oKiMI9A6zZ#9hY7V`Gnb5Y`^PES?Pc9K+*ys@<4_C9U)@v@;#=>YaZ$dBg!8;cUV z-JVSk?GTIkuKsB_46($ejU?f3ShkZ_jc4jjM1OL-wj#&7$S4;x7$(Z;5WsnvZiN2) zGKm@>I6Rf1K$3>>l%V*H`*S>1zt{u5Pm(RvRnD zq!Yl!Q7LDIQ7%Pcdk?}^1`7o8L1%#BQys|$OR_&N(DFBTm90j_R6RG0f9VnKsyG*U z6?ph-Z>s2X8THJ`*n#Z;(4d4VKL6>WgAth@Wa3WcdRJx)B;mqWzBlF^718YqnnGyc z;-=DB%4gJ~!k9Cv0eyvVycONrdNAjFfprI{uq&inT2A@ul=Is+cNB^~e`ft~M*I(6 zc3I0<^vM~{puv#)&R{Ijm71G}6Q2^4H}G|#5`Z&1C5tkWI>k#_dh(+qzw`>K?ihcz z;8l$p+tLJ4eQ*{lR!{TSz}Aj4)X?txQ?qn*lPtP{mri6O-$1GL;Y|YqN>aDC#<9pmD;LA=!)j4{WSaQj7wmD>ZMCUYp(x39~W4akh-CS$%9v z<}ITv=hzC^Jf_B+ig0hqbRMt|O;&lqEY(GHCvW^TZbHIecg5C{l0sI1Un&9<1oRYl z7WJ;ajI#Xn@|&3&zPjI?Z2k61 z1Omcmf-{LS&_e;uN50k=5I=|CedKA{APN;0#Q@BD@1?o zH4CyuRFrJ_9A5rIoZlFzRIycfZ|AwA{_bnXjudC0W z1$*^zj?}-(GnSLPrc^amsWtmzTk+mcB3m7hy%QJs}Zhsrx8gXdOK@;8> ziu{Y4>lYPMVyQe-&*JCJt zMQkr(aZ{|k%FZxrF-Nd-*mt#7uvS~^v`W-S^px`ve$O_uK9hOcRo_j|xe?`lNYoSG z(WF3$u)lNO>kqR1P&4@9Ny@R0ac*MdT;{1=F9PEwS?JWuesUP8dB#oPdyDavxmlh;K|z` zI}*y$M3wE^aL~SA!X;Aj!^?y+R_oq0v-Be3Gq>v^IyaA|PXKa9;gNrzWAvZ1+5Z?G z(#(3uUc=GWdtkiYpdF;$bsF;Zdj`RAch`o)hCk5-=|)p`M}*;NAE)m2XRqg$b?mYA zaQnzRp>#0ao5ARwwYWy-R#;-Gz1|&+a88-xEhfu;psU*I5 z$!+W?PM_^f=m9Uw*=ab8K&DG6icfOivrEBcNOjgn+g`mR@0YQXx!qJQ)65uiSTs0} zJ@fMRLg161e){y+a|HmLy*$G`&TAQ#Fuof%zw-98T%$tyfk4w5PhZPUD$l_~{KJtP z(eGI%$=FKq1Maofh*MAh0{1V8WfA<)1MTCiK&9U4UhdeQsWIn$=^_t~RO$PaK2PcSAAdgoc+ohaq_G&i!M9+r9;6`i?}%E{|BR?f zeZ*oz*B6+(g8A$9y8@mnhnVVp9!!9%W(q6*s3&XB=mfwO&ENFqj|)4+M;^+X-%U}` z?t2@c*t?f}v@a3c3^)_}!@nqQk+*-bei$t=f8k->E1apxH~|*7hGl8mKKKx%U8vS% zPYiMevEm43F-h*Xo(CZgI_HPqucdz7&;z!OXYBnlK&lfq)h9~Dy41kPsYyFcOVG*# z2{+|a3z5WQ7HE9oZKyx4#%Z7Hs`^zAt>X&n675{cQRaB)!q#cLhV#z2T7#b5>TviI z?Y3&Uu?Pe6GpE5*fBURGe6-V3Vv^Y6xF71RlV8mleigzzB&t&Hl@W0Kf zGnx*7=mYJ>y@7=XSB1|l2c@<2Z|hg?TJwt_rW^B=cmD76|8~qL)^W;m;=S$akCO_% z89eS?X+#FcMf`mtuR7NMeeSzE11-nrp(H6DSM-*Qo96sOo+T*{DBkK}I3Jx~`ucTiYs7>tUFWg+@oz?`vU%LohZ~r z3hrhOXVgpltBD!sj9zv|_#p9FV5rUWklsV3w{JLf*WUl@RQ#E`tfhaiahKAy~-ILk}&V0DkRB5UMZjr3|(37*4h#{ZG zT+zxx=s0a*2*wnoV%Qoqz42)zp(IS^f#Y))zuSTUxF#v)^M-CmlbGOL?5PxDjPzF* zuw>mpFUhvo%=@3_?zKs*KUu37YEbJTrS!8F)Gy8SnN_EtZl>M@hy6D3qEfL|Jx9gm z0fr?>^Q|GW)}msPtna;M{GFu;2J1z`vuhV9<|wr-8&ENlKkGch3#nz@YxwsXH!eQ* z=ak7ioas9}0ep^iwl}4F^}vOC4?j^klHP?vJ_gx_vNdXX+P%~?VFtAIGzlHlgrE1O zgh(9muSC$Vn%^8-zG|$p#uFfZ#8(S<6EL%OsC$@x0(gObX?>wRiFG zDJLgjXQV47CdNSIrAzbIw@FK)&z*e$-~4sn-}r<#HUkLVIPyXW1>! z$vLAnuDW;&w%ZQ?_ZjN$IG$vv1>mXOiVptLo9OgMu1+ zoR-f!c_z*Mj7niMS$U>8I`?p4_Ud^f72?1LPZ6w{aiQ;dOkKCEMy(#$6SoDzDc+{ebsmSvkXFTE|R`EJ_OPvA`>Wzl20 zu%DdbSLYt17~mv2y`V5jgZfO(UXoZI2!ff`WxR!8i)2x_&!ZMIDI;Vb^j<1#67FsV zMdVsrcg%Z2{0Ts!!XboXw zlc8?lb`{x(N_8XZ<>G<+btwgXu`$53SNEe5I6_GESF#>hxh|Ry2xOq~X7vmC%?&{k zZG3OjK7)Hz?VZoZUulTiewPO+FZZeKcYyfKK?-@@u0>+3QZmnVgiQyRAS!}KW??(} zRn;}+*~PA>zEWWWDoD~hImVnzwY@d;icIFPR5~_ocyY(oHNci0?*gJ7+>3u~mOq_g z=2R#0;UHtgTQE|Ve!jubs;We`Fuw$D`K81#$k4}KpfIhaM?V@i?6#9sSCjK2=w^xy zEJih4Svo$0DmkAjAMb9(=Z2D0t@VHUQFtUh&?nF^;Tnvg`8t(84vh11a9PHx)~Hz> z41;YJ&!@zEUZCbfOoR1C=F|p84c)C4+$L`z78W!IMXR5y&eXT%QNw**jw5FD<@Zrqftx1>N33Ct^`(moO2MaFJlQ#eZRG)@<3W~7 z5s3Dx*sM`-lm(4iEF1xwtONg&ODJ$xmzd7LmAkH+?iKPD_x9eiq+Xo|h9bke9lAd6vX#8jh2D zC?~3VID4=O{nO`}_zrY0HYLr#*JF3JKuFVgP`{N~S4%3?H4F$>>3Q%AAN}JoN@(B& z;4P*h8Qn|S`f^%L+i>ua5b5=iq;}!))c>jQzq<~iqXU;ij^u1^dcJix#rDM3fjPL} z_S~&Th-G{><~Rf&+3iTGd*!N2?wi>-X;m@xBM}K&zS3jYSGJv7p^MvSeH`*(VIq^T zzUOlA_wBf%`7RkRv}VHGh#1-5F4M;8Z$*sI_-~A=PMB$hql=@Xp6>12*4{oUGM+a% z9YByKwwZria`58+l>aZSPom`D_NFT^x8Os5(S(80qY#TsO3r>0D-=51nKIYuT}U(V ziFw7hIen=sq>~%b%^PF0Na8!^-|oEVC5c+7HLguxL+6(2sl-@@a>>8n2b_ui^ru;Q z^ru<5G{ZfjXec)7>~*N;=Y3rCU^}YClA+1{1KwzCjQY7?OFr%{PFzFV4}CM0M*_b9 z3C7iXO3S)hn}p&(ma1ICekO#$0jUKT+$jjM&ocE>BV8?!_er(Jltq2;Sr*$C&hBJr z`U3ZN%!z|}=~?b#yy33OZFW9#9u^Kl3U-AS?>v{{gCTTmx7*hDz!Z3OkB^DrI+mDp zJYE&#Zc3TWr$hzf>cr$ax-?@9Uzn4>j8TRcg008{qZUV1TUirAN}G^kKkb$d7?=E=orwuWzp$Z{Z0FJp^!TYVO+4|&QVL9^6b%5 zsCJ^EBfxR?*CI&Y4Qf`>b1@{Vg0T!qM&z4UK)vJLd`8#OfVR#Z8Nso(jLHB^ao*G( zn_`oGFg8qw$gM)H!f6g5Ob}@+CS6BG6?B)vNAxSaqt=7fhw&|*MHPvN4*zSMMgFNO z<;*6mU$v~R=i(3j`EEPpTAAerj;kh%Q?}`;OG%@qu||k?2bOef()UOiBBiphHg!j_ zV5Ki%%NY~}e4AZ{>28zJbx0I!;eIBY+76`@xSA^%)_SAj>xsCDdp_pEqBsd}_qI;2 zy4O<=pLu!A&ku;ZDc}Pff^27t2nMVkMW5W#%=5`#L2l}%d9I}IK|~nJ4U}n}<)wp6 z!>^A`1}36@tD_S|X=Ak9VjJ@yZpmv|Aaf_MgcIpj-fErQo#2?ffkQWJZsiQg4%>aRzyAhA^U2taGFZ{@$7IQ#sQ`%q1uBaXsc zrH-^!UPVbjQMHK3u`RRm$*Y@qzDf;Q4hd`n|mG``AkWNedLgl zbCRtj8XDDP1?YKKl}5z}_N0IAw>8UYf7#(;t@@TXkLpU-ZO-zA6cwr<6R>O#3dLKP z2D%z@QulpHX`1Dv?r|%rS`ew;#S~=;iD?TkGwqt{%G8<|c{1%-II=wR4#exXyiZ?i zsPSLcOX(+cO4LZ`pE?D;(n5^tu;qQra&A_Tnhck2QzAscS<=jLPY-!B;Ms1m*Q>y< zm->>*LP*=IO#_>|9gO}`(jwCr95)M+yRv5BV@ED!)X<6Q#tZo)5F)HDyU_T|vw-_g( zEO-xraGhw)D^Cl(_kQ&J<)8muf&52KV__hQ*9U9h*(@dRRVEmXa%2K?!9AuHn`dk* z2P=<$$KxTD@i;fdLQsb6{l+xo5ozCv%=_~=xDzf6n^UlW6d%?j0#o{(G?;BZG41Yw z34xqfG7%|Cz(`>c80~mQ`Pzsp6%Ex2vX~wel5@p704IAG&MWS=y{8d26P2?FDI{27 zRm=8rLUY z)%atMi8_aYT!>Y-d`XYcZ;}8Gt$;ivI~Zk})!x zxm?*NfZf00j+&%`9W!@R$U2d~%?fLMGX>vrX|{sr`(>60f&_DjhmrCNkZWqO2|KUr z>p$gbAHcd^Jfuq^iCgXq`$&Ojlw)rj>*b#cp4O&nKvf#qV_KN+8)fy|)>jT@v7Z%mf>Yz99pH00;(Ie#)Vt)PTYwN0Ta zbrNS$q(wj)I5la113ECQ&xo%2f_FD1)?uN#B5lR4y3zsL*$ljERj@7wE=|xwiC67F z-MwTQX~R|3tB>FDA1@l!QLA1NnXZcpbJoLOA_X(+MDZ6Vnho*}d>TY)iCO(+gZ?4; z6}5QxhsHHvBt3=D=^OhoZ6YHywl?81tA6`C)sIVFODh9QOP)BrgeGu3N@WIwA>Rj2BZXiyX`#$p9V%UNk=5x{8JrdL;vCqNaeeixEV$=9FiLY956YO%l4WT?ow;n4 z?L5yfPIP^_R$L(1HK)A){4}(`E=aZzR8U}G--elm2I*_=I?~#PM=ZY#40Q8QkVyBm z(lOhG>#1h-pUXV8zsG3350*e_#A(Vz8UH^FxoOWb@P9y;|3cq+HnD#Mul^4R zjme*|Owd2j835q%Uz-0V$y*!eps7{wY+4xm@^f8$PUlRdEWE&kfE_N`UEg3zi268l9oIQl2IOIN?2V_ z8Ysv=K*6K-n^(YWxz>&qJfiRZfJsBmX8Djnx2vzjj7*c`&tvXX#AF0yJa2oyN?9yk zcpbrfL~M;Jr{WE0AMMWpeNR2|vXWQ!9}{2fmQO8m|DBCQ)lbw_-pubIY1I-{R&<`O z&DVI!3d*~q@%1rPqp7rc6xX!V02GAzX(BS^-s(Y@Za0n%eV=GPQC$m?Ar9k*s2ks* z6(qn68>OfDOt9Ji#oL$1vz@hln|r2TI&!INrWP?hoKckZIMI;QBqs%`)=Oc_uTW`^E~t1?|VPb`@C=daL(_<=X}oZ zEZ24AI^XMic`sidN@#kZ7<92?e3uy6eY1}YImr!W59u}xsm5Z+IvH|Q5?aik3mG>x z2*1#8h2Lm20Zn70Y2!S+Ku}#Z*zHnQ)VX77RWGoh(1@*0SNyg>JzZ%$j-F>~R_+Wu za)e7rT~S4D3#XT{@EMp8JSbT=d#D^qu+Cs*xF4J3oWHbSVxE%jF5=1Yz85SuR=XE`q{{L7)VqkpYDC>OeHPo&pf6B*+Ay1TzAH}Lns~NU2t7;3Qyd4uD<+*5z)@ryw!QQRVq0%hCsTy7W&3rHq=t;O?e`A!%tsl0X^QhtOaiK#n!nZmWm(s9|@=Bl7Yxa(+7 z+Y(C4)EKGct=A%?SGN^1aDkvB*)z*Non;on(T3~JYx_+?qdOloE5H~Fiu9mDP;m+S z>ana=_i6_(OG-_}$w9*LIe?{qiY8MVJ!2Qp@|H@vwq3HS#RwP;w|lFQDrvviS%}p# z7D|iJ3-i$N;z9YWc@~+9r17(+E#m+`w#vE%Vb$}I+mWzym!a|eg%;<>1&MdG`p#eA zox<|974L#|nf;Bw!M=sR|4*m$%*B6;SWs~n{QVj=sXExKWw=1lwPH9Jy==99wSWD$ zf+oSo_n$)P`S@<^imD@pR1Z}43RF&0cSG-WPO6 z;)r-C7A9OtU|L9>SeRPQiyd3xSs1xUuPzH!+mGD<>eVwkRI zAWK*jFDKZ}mnD=|an6ok*oG?)OS-+y^&@}Fp{f4A5tP}Fd-a&GrnuzhcEkYov8MKEm29)OW-NJlf;E;^qO zvTAbID|b407hBPxdAOg^@`_0)9c!DmK?C)Ta1BdAlHdNycW&8xq3@~}BWgr*ypX5$ zV$5tOr(5CytXwY;QlXgJmgw#8pTBi8x0rJ691cWu_Ao|;*W*YS>G*3UZ&ruX`bx)~ z9DCEqn4vC*9%b|0p!zFh0wXU1&f{6K%J@7IR*5iOR=yNLeWjPtGg;+D&XW(=F4dtI zPqhe{_Z?J7h-4i{CF^8vDBM;w!c^l7-*0*eKv$d$J$}k+%E3TE#nP9$6y)maRq7FI z7wB+hU(YcGyutYdCA@`Fr`wpvp?3M8{I*NjH=&x)w*8BH+DsejcRWsCECPA$#Va^Y?g(e>a(bfB2s}IO^&$rcm-&_-0tDR`Ao|RZyzH56D52GPus$7(<9W1YCslhNra3VkL+;t`EBvNP`{qRlW;|}=9pXYWb5+Mr zbPvucfCE*@5WG63)cPRzThn`x=9@{iS-ZQdnytZQo9>y_p0nh?>RpUMwRl2w-Z9ax z+{%liw>ajtti}sI2@`R_jp9fn8?YL#3qnhPxz8BBhzatZEm_du(z(9w9j&%fq2r&N zeS@4Sz)JGjj;0~C%6CaL4ZpaTK_H6!CLZu?bSyhXpga6VRumkeJc*GobvKB#eo&JL zc~+vcM#Y$CcvEAIXO~(>+8MqVChmC%SWJbL?VU}2ZcpO{kKz?WOW$+@$$9vifGnGO z6O|Ujpr6(U)e@br#LPKR5iwIu^pel(Gq9J!KNi!;PhmAFwT#V9^=IQ?t$VWiuN2SSAtevSXaY)GW)>jsLMNyVr;8{ z0GkTwn|ea_XoSfx;%mI}mg!q(!8~sSDLraTfTP3{h^N|WN zINnon`SHF!X)a3p=CI({s_hn+%<$3(n2o%*)J;t9b?o|7f2F?6X~%_vrX#E}_Y(mQ z4%VgEJHPP=1g8^KHA9LX(x3 z2)U7COUsevsq0ySPDg$vDku4loBzj&{O$SDKj4)vUH@j*{;M%r|E|w3&%d0}Ax(g8 zWK3d}FPN{soKrsedBl|r{6#S3b3Z#>3n*YoC)4HQLq2vZD=c)9(4g zL2AoNI47$#o;_im=#=eIkG7|8eSO^M&hhWQ{4X#aN`mPaxRv#`uZl#Nh2^oZR`~of zgLj&huQI5K^Mh6$4X_~p^_(ppLYil%STtNqy<~|!wNAK%(JscQWMm?Csw@-h3qxEs z2T*cdG$egWi(%G2%JWpse{~!OBwltij`bwm0=Yjs3C(j1vlIsbCPv5PfH)*%z; zY;dO2g1Lf-NRy9_nNxX1o72haWR+7Ov@XE6WIip|h91(vzq*zkpg2DQVfurt0xO6; z=7-!@A?tyMpIK^~5rColg9KGU}SGvH*|1S!H-(QDUet8BRP#J>st^#3%?WoR(4aij(MD+PDL`9w?`Kk>H71dv%P34^#8+~itOvV_g z*5FPu2;iBU-tuyRh!4%2H|3{Oi}4!~)q=x$Uko-RtaQIysl2RftF>3p=+u01LaP3LAMoeATvH}RzNOYN`kusodSO<0MP-MQgTfK)tK-jEY z+ayVN(%DH;&LUKFF=SbJ#saRqq1yO9bQ20<>4?_9MTz3>(I-SEIq~fG!#d|;r@ZCM zZ*XhH!I~HlguP@|sqC=ibK3U7PYsgaf|jzMje(Lfpgp~(6)KxyTj}5Y*-ZJL`@zb` z*V?)DE8qH2_lDJFVrg`tRpbK||)U*}2*NiSUnw)B< z`{zQEg6#N7b)o$HmKg12K(13G%i)TK;d*ERlW}zIMGf#iWBL?u{*?X^B@BieYm1LI zQp0zhRuc?%aoX+n&19HsuxdVkD^KER`>fb6!^)$n!TXsvY^zO2MPWBR2a5+WVu*L&Uh1fSb{LQ28JpTU-~VZj zZu}v3_8)Q2sy!3Rzw$i|H;-&_pG>8gd`P|1`&)7U8Q)*N`%^C`KH!b{>rL-TXO2)W zl#_L30AMgf1gJXZH*Rmy*!y&ss)59S5{bto8ldjoozKPb^K*>}mG!Z9Ab~8f$>_Cw zt|>O7k@lCzp{QKBHvf9`C3@H!Untz*v(J=j1xTdIkLcj^*hU(dzj$%9=_xI(<4l>y z%O~g-`W>RF1rqMoBJrK^bW$$eYtXl#@m*Hp%#t1&=g>Pgl*|wTsGRfi=HGtuA{Jn? zOxFAa#sV-}l=gd*C4yaRpVtVdP~T3dVr|vO>;c7S(<9Sst1&~#Ni9vM&+5?U2dy#c zTHP5%Yp)%KG_GXw!e!|*L5>+XS?QDQkCr-HHm|Dz{4tMF_v@zpp`DMqRpCm3Pk!oz z*eYGNAzxV~+!Tsl;4$qw{C9ThanZ{psjpol9=s4Xft`3KGH!i^v3zQr>X;+7OR<>e z!CnRKj(q#ao#^@@E&s+etAe_h%DV0v5cR?*vPh~SLHIvzIJ9A`{p5Nk(lw!cWp!5$ zu^};IR9r}y6HJUdz_$Y=L&WqGj7|_*>BXDeRWzZ>hx7j zl(VUy@dIUJpvj-*>X*m@ZGtq6FM>@S9j3A-3z}Pn%{Df-KU4Y-_V==TcH-u z$ManMSN0a0`r6*-N4c32_7>7n`G4$qlJf95+LdQFO)^EC`#Cyvqc!6LwpHki0`aWeu z-Htp_AG7U!`FkFR_ka2G=;g@m)nnWA_b;CW_D90UBks2R<Ie(!iCc1~CK&hY6G zZ`d9xSI7~6F1U0(W|yOo6{;Da^k$$YEiE%A42Zm$Wo!{xwBDZ8ZDw8)kYKUUhQ=Sj zOAm|Ow~co)JGkHOmlk>vj)v$41BM%lW_2qdeB$3K;R!|$0nS~dsm37T)oUy6Ic?5I zA9k`2BfF=53lkzLjJN|0Cq4fLe;9Es9@rcx){^|Rvhufj|8K>>)lo@Faqo?6gNDBs zU;ck;5JMu(0H6d^u>R7=Cd^E0^{JOQAz4#^N{x!_`D#AC@UDEq(K6i77`u86BBhGO z|H{`m(l>xRFcouKOJ(E(Y0qf}x`UCF$j>S1AN`$`5tuT28=UkdqneFkwg5l?R^#Bk z5nUa~N*342#x$br;l{<+vE~S>MAlb5JJ~E|vI*KiiV6{bv?_I^uQGlY*dFeV+`M_< zAVWqza`ixik~kp23R9|OSSN9_TjITQRtss+cZuNYY*|w>XCx2C%PLN-bhkR1?A^!rX5KDRnNwj(*Cx1Blg@t@F6{BIBOaZz!So<`)ad=h&vBEL$%S`)n< zxzBaq5v~ZfJyM+!n||xM7#E8Hv{5MPPz8)SxKkSr?xG6rsx-c-@EN}wx_Jl)(j@D@ zD9K-gC%aFZWsT43OutuL*fb;(4on4;F9&i|-ClHDt|2N&R4PM219O{8u)KEfd^`!w z9LnFA8F{T%zjtbjw&6i^1#h{uTGhvn+fEqeZyi`acL)J!P4r1)!8_wt`*(lk>u;Wu z60~n-^E5@WQvOa+4M7onh>KSl?oHTK5@wSRmnv*oBsopGL!W5dl(s?N6_>O-cai!T z1M4SinH^#A$0p5Q_9LHUz6_}2;0iwZ6VKk}>IjZQO-G@!dasui(O5iVRF;|H}9C zR;~X!DRTSpwDNI2Vi=@;mt`1s6wMmz>mAr1-%zZ(^cV7?@63$n;qu;j?;t~6ljW%X zs!tD34UroQ^3dBYv-+A=XI$dC3u#+V%YK>)>P6r{O;s(O=l$oT z-`wC6)tk4CJq5y9Eww2q{jjx*(ziQgvm=u&Chu4EH)4?a_^^*}!*4z(|5>b*88Ph! zvCtEjx+ZTG<<>v7!CqV)%e02;Tw$3-7$pTmjPsLb)ZCuX{WNm^ZnxRppwJT z!_LdcF`+h2Uw1g=MGi`?HCjDsa4Ui=Des?r7-l47J0@t(82wBOJPCL$V8VQ|M0YFpGz{ zIHa$NjO_!_r||+g&j}}_m6++ATtDqC8iJ{Ap>rf*Hu$31`+_OAr5QN9JsgTdVxy#o*kw^luDY^+!wykLF8i zM}7sPqJhg#H%~eH6C6=;$h6T6>%KrMuMne}h3${O^7U||Aa*-x*GClgRYwCI45|t5 z2s^*>$*iSKhRtn%v?kb2`WI(hy+n^rT4Uep$PW3`**WvXZ6$R8$cWUWM-HeSq7Tg6 zHorckTOwVPH>F4q8Z2>)1xgvp#NwjdPIuzxx{&d!kyqQw-t2b6ZFfqRPpja0fQr0X zugFf%o@jY#$#g~Qmki@@cghCm+rc)~Cpfh+O`nWD${O#T>%x2{01+#(j#?=KoZNEn_ukx= zrLASd>pX?(GfWlYLTu3hS}e~QL_~u-dsbHwDWYrH4OpIadni`$?q!L=>K-ng^z%k) z%EowwYyFi8A|qK6t2SHvLXinmbW5_;b$w*%Xm3v&KBLU`N$-wU&D4MJkt_^-p%!(;>;c zZomF{rrq6tB9*0~z!9&&rA{jB3Rq0ndOHm!FI+tagFEIqrhy2Wt;!OhnY`(Pc02nD zg`7q%Ejk8otcr*zNPpo|U+C{2^@2JDREKHj}H@&JpSabHCe zTyA23iM4i@eX>XWtn8g2-HUgn0X?&hNeFA9#W7C+=ALGfG>=Ubr&88Vx7`SWF4^GX zwFk$of`jeXn!f_iC+jK!;-ig|!B82yKrQnvt>)!#c_*WGQN=kIrC#L=emrFs+PvpX zh%|XeApFP+VfA_$xK@I(ZWGKg=ZUPPEaemBJpjvLJK49905B`=7KPItcV0`DHBd{? z4Yl^q)xp0hV$SjkCQS+)hQyx0=CWIj0TcY*xysvO#qCk&s(_uN8RSh8Ipy z?i^Yfx2b1?J7ZijzmVYIVs{kleFyELL(SmQ(Yy>A&(P)_k_t+i{kDNixx_hSRdljW ze+fs_1UcnaSwumXkj+)(D!T7f9|=QFObv@R71D;T@yzrmoPh8Dv=;v(E`^oG+T<|` zKousZ=pHEf=_D%~gmXy-J6iE@m^M&%@YodU<3S0kx0hMZqdw5TO=Y385=zU8KTjpi zbXiF0y?F6z;l+>nY2^+JF+R(%CFc#U{H<{-?qqtfcYy)z@g8e(g0|vaKT_I#Yb*(p zwowAY!E!M)g<7QHFFnEX4xg|+X0)}Gn1uc1Fm9g^PIv0I^THwIBNOYgS4L-OU}ZaQ z@^YHoNV7NCa4vAv_af_EM~>Z>4eqyUb9wCfutjek6=}~NNVTBkg;s^;1>f$w5%aAD ze~{k0B#1cFF)avO?(EQNTry>o-bM`xy7J*_^-}(>ZV(+^VlpO$=W1jJb5#@ zQ<88zp41>c{gHBIP z*|B^Kgd$kdy_p$a0GqYKEKXC1aD>(^@^tlubInr6{FsAqU|;|I%k&G|vg;N8#>^}S zmomAxQ$SM{mUGwU@v=>EKv>SUnj2HG16r@#WBQ{d)%&wAo)K{`y7j?SI&5goR0685E!JKUY{%3bxXkC`W z-P@vCotBwmFtvlJE1EhFdeJb^e%qfgx|%_vo4RiR&)>1Pf>$OwwnRyoAmgJ8xlg#g z6TQy@vTT3mh1+40rk*)_ni^SX_EXezmx%<-08!BqLCYv={ zSxXKJ%Q~Itt&UJSsx6CkG?ksIsdZ1$4zH^+F+u8Zcju+e0x#tOyad=BRI4K9?Xor< zPTbZU3-P_44(s&SQSRh7n zr7p&8LlqfLm9p*ERx{ZmQl?dR#=SDPFG>RwP^Lyv(0)=!b$aORjav$L8Xm=Iv%^Ru`V8Ld!uTla{O1m1BLCN4HywD$G{qa37d`SH5HeX zM)geIak^7;_Xwmw1{})ZNsozRm*01x-0PKxmV);}9`e}CSVA)uJ-#MAroP6Yx0%Av zQWGHwcDDT$Mvx7>(roux_mJ0{kSKhC4J}{KPT4`eN;Px@^NLx61ELWC)wC(aiP}y- z13Z_Trz7?9fcRwkXS_*_?X8Y1uWxYkwi&Z@jLRewnX#@ZFI0lP#)3}k>CJ21@&M7c zxHm`@a9eJLBVS z>;AGsT(&+G_J@gcoHZ8}R$kJ)THg0oK=N%vuF>WY(3=NHz(9eizE+&lNg z<(5>fZC}kY2gNef4(Ux(Igj<3g_}PewpFVjRkZ7|i!M1;0lo@e)r6BywpP)xn#%J0 zE$zHKP#PtLo$!;eE zrp`e*64le!H41FsFkxy0qRtvB^c?vvFYd#9$xvw9VAOz<{|cUX&7Yd5{q%ZTT=>Pb zuqN&Dw(Y8@?7g65%8{f**v>_h5!CFmN{(EI`^K_J(Aw9lVRd2%mm3}NYZ<@rO(4Ht zZD_!H++&on5`sTfd4@f%+y(0!e^q*gQfYsH&qqssEQvckvKP##;_QKKM* zvZg--JXaVIO6|gNZ!RWuOxbK+68Je`tKDr4ADZ-gCx!p!tS;)^O6ZW}VoSSx|9awr z#O@SN?dsNbIizSk>DG*CJbb;XzIT{+?r--ZMprWXKjIOqc)#5G$5*l!c<9oDj0GVS9# z5n?1V*5a20khOn?BLZbg8tn28$!XAERMNFc9;2?I zjgr065h=m4Q+XkJ*Cmh#?*p8Qzk?OLd!gPsgx!CS8QU1c(wJy*Z`08^+Vh^fwenQw zCMkZEx@>w)qX{eG3p2AOUKk6(yep~6@#SikRm#OkI86fZa90drwg2_o_@!5>4?Ew@|Mf_J5x3%2mV)3>!&bl}V1dPp z1R5SK!>?s-Cg3#K0bQh|P<7A$94=p(RUgvvZ2k)SYpI7j^N%P=tJG&Y?@oMFBkMa3 z-LFD1K954%OEH(~0p?BD(k4eD;>uB^%xq$hw>yB#;wIV57;zJ^`a@l&2HpiiqZW!E ze9WRWQ|1(24K$f|2&6ZbsB}tqyz^C zbDlr0zF8;uJr%I9MH_5eDz$vW@XaGSSP^u?Oo8&=*ifDsCoT8v95` z%68VxAhE*rB?T9U{BPpDg&Pd=KwEQtxM? z>Hn08`H#Z$OKVW(VzLPnkY>MHRJFV2V%JTBWoM^>D$6hc%oB88m*2a5t(D2TAEZ1& zK)D3#dOLn_a1r;dgl355_%9{1<6?6k_W%di9AGlKiSOZUOJx@?+c%IuF*I}iQnWp4 z6^Wjs>O5(CdgbL{G_^OJZMnnTlN!(&39T7YFIy}ccg%n$3lw}w;OuHanrNc)`9?*N*0kZeG2tRbOBXIo!STp+f?CCM2U8ld130~ zmt4<@j~xK0KAEq|SOLrd{D+@0OPCqWfF9JK2A(~gf;*KKO99LJS_GYKmDv|i`|G!g zBZ7UFd+#GBhO(c^)%iVIkA2!*WPTn<_pq~T#JS{hfi|Tk@;9G{YZhq6rcAm)7koDM z^{=GlcmtAKAlkBnaaQAb`pC(;Dk`NZu2Fx9Tdh}Iv7c<* z5uv1YkYSpqPq$9Q98h!Zl5S2MU{v|EoZ;d*jzIyTcwC)_p5$4ThlIFGo6i9KUBY!x%Us0(LWqGwCcfrA7`zd^r^Ms z&4puGow)hx$lvd;uj+qawae@6?}h2}yB=?ZU9V%yJANxVZ+!L78u}9U4Po-CHo0hn zPHl~>pEGZK1uKd5hEu(X1>w69@6sTI)BK`*=XF+)O5EGBwX)&5i(A`ls^N{V}EB!ixkzUb<;loTZ+3`3^Xl_zr?`N%r#ZI6jynCDlaB zYPOl+WenwMgZvLIM^|cij;@BlCwZW!G6NkJAB@=>nrNk`8JsW>tG)f6CHupS^0?^Q zT_(0V)ZmBemyYIn@}`WEJez9m;O^Y|z0>C^)x|O%kDY|$lxtcpwFj1mA1&QIR}JpC zvjVJ&^$PF;jp0n5ZwJ4fq=oDD0;Va`S7`H{Vr4&@F}osRtY7*1_SapHIzQXh5@0pG#5a_tI4J z^16wk63d)|9-*2J=HQ5XZEENh>GY;s~|XJ{cZ3M;adg_$F?zu z75AILWc~I=;e2be>KEIe0>V|=8t@U3(=!eYwY4D}z16gch47Y^yJpt5bf1lGPurGU z=pgdyF_0BI=^9%wVl)g8%MVeCvPToSYVg@UAXwhT+Ur7Xaq*Dt3X2E-)tky*KAJQ(Ce_TF5cQ4YFvb(Mw)DAcc}1h$Id4PEl}4NVmBC7$ zHbk*O3=n)J*G=x8M*4L1XfO*ZEG#6WYo-t?n8go3$+oxX-aGv4|6uNXsECb<>}Dq8 zk!_BzZhxk&^;_4FyR=8;^@~c*!jl8CUP0&7yscqfP?I?8iTqe37{OIH15pt>W62w@Dxw?U+&+rOkB? z&(i`B5Pv9cC;)Z^KC7xpCb8Y`ff8cs{S)lNvqqt!o3qJRH}`X3 z)=xJfX5!gGOY+0aBsm&R)U;N(?%I0=BHRKukZLE+W35&)P;&#^a(2Vjp&J|fw4tiQ z-i32aaKpTrxT6pvc`v|CWE+!yvwvfDY{$k22JQ`NV3+6)FZZf~lrD&=?G4QKjvH5@ z!CG`I{A8yAwW!MXl2vQE_PnmyH-9t_{z?5fWy&!_2@l4YGH}OUwdy_jcr!)&FUS2F zSF(}$1%O1NG!NvXH0Rr_6@IED)wqfga5(S%YH8HH-YU&3rqQvL_Uw%t;;K`K_7ykh zVRr7Iz#tul34t=+o~I@vV3=zK7gyQ0y|Y2&L0>&dRX^0JrdkXrgh$wC_U6sFy;Tbt zb!x88bmn!1nNekJJ;sF{BL~cn_z=OgJu`{ab#X&r{$fMWHeA`(a~+sDJgw0_m@BlN zr$eG+&%j$d4CI&ZVUcGUQyQK5aAzp$M!QFtF;~aCrLK62nw?a1@H7`fz1D<*yAE(0 zHqhQM`K~+@8DOTkL)^uw%(`ZUAG|0uBKZ^v>u+vB=oS3D=H+BLxD`lS+w2u4Nw_ud z%5t7Zg(pFMO%pY;*N?iAYJ4Y5f* zs83^Mf6X9!;|6f1Z8z)ca?x^4(lO(=>P%jTA#P3qdS)u)Jq^%1c}bhQf96#r2A7rNb^G(VYkF4(fSV;XpaQ?Bz@H94;GpjwpZ_N}7!+N4$0=)rlHxY%#jX#k{{fDc*xPx3$F4mI1zWZ&!h?Wbi&TM z4hOCfQ$ZLxg}>}uRNq8u>c{wpzFj%?M zZQTs73!z=PXrQvXyU#Z7JjsKm-o^*rdTT+#x)KX@1oP4oyhLo$!iM{fr6}~m#~nwV zy0Wa0pfS1N{Z}8dFt@q}Er_r%oxuhILKHL#EK$w(|Csfct3@7!<&XNee&x$zMV1)$ zv(QF)6j>VjWLs{Cu>!wjoayV=ckl85)Zc!?_eJz|sc-arRb(CO@SARDmmC(=x$_z} za{#LfQSy=fx(5}9F|*3kmhZ}+^iJ~6<#d0(O7I!8x6UYZxt0C7_GSU_yH{lTV#Q5{ z4P^YfljyQ7NfwmnSY?Sl$>V?hRFu~36GIxNXXnw`c9%a!5su=h19K>K^H+4Wd8jO- zju&6zYZ13@?-2P+k90^JR98S5WpD>G<{C&C$8@Y%zthz5T};w$YL3#IL6r*G5gICZ z=WZ^HOE2aCC(4zKETTXCSU2W8@8}co-m}&=Jun^XNO`AIH`4PKX?(UqN+r&%Riz$k zi4DU!F0eC5-QA82$R7Dxuonn{6#^(dRt=YKT1^T(Z~lT&tD}a$XH0~G=AHS6Y@`|1 zrI^M>IXC{#V_WT5CFNx+RK$q4DNwz55R#&D_TkKy@AR*HkKrV5n8q4-Y4h|eT7{T7 zt;1@*w7@>A7wZ%oQGu0X=pI|;w{%odIt;)zQN1reV zU*>oKz-XPHkg~l*Q6r(EOh`;NrO~t55eY^Bp+|C$fdZdbqs6E% zgZ!g@Hj3ph1=SJ({#L=oW0x8EOL`}Lv6jR_A9ng#u|1)Uk$pRVZID;)Rr851)$nTt z>G~7rekk?QV1YB1mgVsQ=-6(h+D>WKID&VbO%u*Th9GQ-Bqa&(I|W*Xj9JK}Qb)e% zRE@2sW+;*fakQ>b&}|43C_Ik@-3$lu42R6HlX*pX)3jO5jYSd!|7;${0fM1R3_?Ps z|2rw6i`5;qKUx&|mVAV+V2lB%1+JYRceafMEJKULE3w0xt+^#1=PoY76!7cWP5!nk zSO6%`^v>Ardy32c_G;>v>}Y`lUTf9$4J4u`ahq7$P>q#VTutC%t!+K68U{8g$GP0L z=EL^s!k|xY;x>6{`^*&;r^I%clB!7AKZvy0gU z88x$FB6XOXM4Zb|A{synddj&}x~#S3(dF>n`+uSl|J;1$%WrrxSgav0`Lc<59jQKL zQ;a@(ttnaj?)8Cr@J1|*4#HY7Gt%e9?c4eu-E@IY-5tr)&h19`{6x51S8w;`L%?gb zwK@QNw59RIFsJ8YXc#!;R5eOL{E`iRFgHETFTAb^p%kjWXl??iX~dmu1LJUY?KPRf zTSXpcEA0pmnrsjvBf&p1C-|Kl8?r9S8Ouj1J*5@Yuaz6pGs6MS5#>#JjxSftOpR`( zr5s@L&z4U@OQayt*TYhSaPDbguimH^K)H?F!N=ju1>R-P=|2c73fZQfwcan$d9YW=FU3P zcG*bEyGgV>iO#Z+bC=S=tPKW?Bz9tD`%9*2i)}ZDoYlt7(d1sDa~lSd80vM={YP7v zoZh;9c;AoCR-rv82e6Ghh?k+*!;a@e=f`%gJ~=&N4uVq$m9Ayj6zvVQA9lPQQ3AKA z>&I0iAyl0q)Ld)rM3n{FFzQ~Sc_0}n2PRdVeNVND%4DPL3IdoRSHfqOVJfo#q)Z9R zIQME37uTHye`+?RlR(t!J%1z3++d^YNxN5wl39DBK2HF9zxl6$xU26!Q$3$IH7wmH z(IelGQHTV^DwriKwa|@MJ_XX_3s$yK(*bFc0S}ZW$`mBtlC41ZzWlnXT+kEZvJRrGQ+CNA) zN|+lFAfscM^i8!b!2J5btLX?jQ=CC8HJVIv6H7taFDrIIh-*t0QX!&?F?$UQr@t-dY!eW&Br zB6P#7kJ0ffA40Bp{vt9#kYFVA5ZC7iD1Nri<12lEA&9?v;8Lyg;{MA$Oiw#S$KWh~ zcW7{Kp+!WyTiv)qZH{R%Gn3O)hXG|2Bm**b3@i?ahNkvqeVz|Km;`>_6gxD>00d(J z{saaiJ=#K>dh=rhv!M}#qZMO-2%H|$Dplx=UQcuHH6>@r-8^|4#^&d%zzC01pjAxn zvc}}T1wNGh62Jv>O4;>1+LAd9%`#4xT_!f}KVND@>>J?&I1y zq816$A>-o%gD`1?zl@j_RHJBIGqyFv!QvC?gNg0COrS+k6a`=4Rd;aRRljyl&af=FFpCcZ|qRFAo zp&N77L`|JE%Y=ZZRpWVrW-p7GZcRFmABXXjz2YC(hQGXYN%i6J`?;_@u6%uLe8cyA7hDQX(LES^*9`^w)G zldlN(Nbi*2^oFwB%d>oD9cbJ+?XHp~e~xLrA!~Tmy=ZuDJ4n+;u5+j!O2=u}Yo?fb zq)qOKdqK-^%EyE1qAiH@NA0(!cu9a-^EQ)~pM9xISQam(u2HQ&h1fM<;V|2#sYy^` zMrHeiV#_ZUcR<^1Hyf%BO31`Rg&vLMO>K2!qBL0h**nKx*Pm0XSUsvoYk8DY>ue$l z0K&*p(N6RyOsT@3@&Mk)i4m&I=_BSAQCcC<{HP>kTen-_Fur+x`RKFSR-;+$0&(nJ zlOyE;9_h#B7@QX0>CkvFEcALI*Yul5nye?^@(|7c^5@I*&!0Xp_|9b9#TmkAmw%QK zByQygu#N=k#~dGGfN^uxprmT~*JB|LV|P>P;Xq>G3}`IUf(m~8d9u2-qjyK>(BmAN ziAdIh50#F}wYJ}0lJ3OxMSrf({*=rXonE=uF{YxRw&|N8@59crxL;KrDGFp|6e8rN zwGl~_UaghjXkZukIHziq1ThS{?Kj2K=>+6WA}Wziv`!?i}K6OvE5zp zy?Tmc=nR0dq2LMCiO8`FT*2&yrh$?&x$yRmtftMHxwt;N0bj+<)7q949KbS;J1<91 zx8N`L-b&l1NX+VNB^&Qe3f4B}>G%`;fqv;w>5L-wve0LZ#VrpHHRt>mY{`I)mqLh6 z)~&=m&yoU-opPs4LujAo1i9R$e5t8a3@UHKEwwn`p69 z*#Uc6G8PISc7!56E}J1Hl{{1$FMr_Lj1eT3Spv9QF2fM|-GFaFiV>U+{@kji;VETm`Z(nCz}*LOl0&+Hk#lFW0vO zTjLoa=~BBTw~r)Hx0=%!GUv**XthmxS=5tW4vt5ten#*$w z@k&!g>5s`gm})swr#A?>w79f?o$Tijh9Q7QvC{TYh$`Dai* zPeNbKY<)Z9+U07TJaj*j4t0Uj%cgYkmAq5|wX`H@*-54LaxJLlyU}^*LEYZ_(ZL!S zV-`VywM`dqUS>^Q4*)y?S2G46j#a9sy z+8z2&>fD2>Ud>!&dt8K;2j|_Hz228nQ_!7SmV_s*H3h3E+kFYmQm}BGd)yogH8CA5 zTlMz_JV)Y90Rq6>C(NZ=BWzwA1vycvkOofPB$}CZ+7wT6IBoX1u6a>I0>1NT*q5#} zn2PUJ5Q|HZSH&U3s@Q3kW|!bdcj=DfJzMW*M|7)E`Y)Y71clYa$819h?l$U*=lK4Ki^cqKP237@z!)cgv+b)B|J=sOboS5 zK4p_TYtBpaOZA_Ac8J+Vd)0z9nr!>ll@v zR!W)|3~#%UYB25|09K3*)arSRz=G++UIo2D<)+sJAEDx)cs=Y>u1|aB&r8bSDf4?Y zWazN_?NNZTLT7X!#mn(Blg1iEeSm!nh>q0(ha+`TkrPczQ=G)LEZI4Yn>DdbFj@Jp zaZGCV^D$8G)|&G!Cw~=UR#m=q*&85q`2#Q6O}|B1!a`Q4QhCzhf<@75Z2yXWX>2SQ zz)TJktE$5Qrjgp}FZ3#e)+c>@FbwZSSCJv1C&W_nqn1}Y_k>rt&AZRc-q&qq^81v( z&& z9T`|4n>25XNE!uO2#z*=o!pg)wKk6Rs-)#to;oiiY2z$oP!X05v_ClQQvLHCI|hln zi4q+dfp2EuO?3AM+V)@9Uq1gJE=S2gLzvrZp<2iwQrC0I$8D%I+Qwe>9>x9?pEZKdkXHGF|R7N(Jahc{$eR(`@Ouk;6#f4fo@--{+DKh99xY{;SI2RqqW~@s&ilG=PQpb!lbU( z^yx$n2K2TB5mv;)_D3u0stayQ!7N6OP34U~5)(7e*qgUkl(Wl}a5KDigDnF8xl%#F z{b|`=cI25t1|C8P3c5tqewc!;aulL?g3-mpgSZnEQ7X?kUJG5!b-e1e^wI?xm?A9X ztDvvSCYyLzH4EV@kjK~CO;j!)geA{s2h_xYXbX>kndw0MO2R%IFx6qvFYE*$KE~Hvxdjp3b5;=#+DX=fD zzCMW_8-4N1g9q=HF5Il66AWlX5rxJU=O`nuu3oXo0%K-$Mxn9%1;3DB z?kdTN(8)o`>Wtu)z(HqaJ3A={QH2lDqQ)6+a@uNXu&v#@j&7t9Z{elbtpZFq!-Cnp zmkQmp`#|@ecRw`!WYN^pYt`wRZUr%pjgN*bQE$Y3*31M=jIh7kv=h;!FI0G0A5gUA z-`JoVZTS@+&7{K5dA?y=uH*nnenSo-5*6~T7cA~K=2HMcWQw*M^|?VBvbs*xZT4pjF-qOM-V=(&Y(67>_!7k$^J657L# zpLrGjxDUCR!E1(o74b&WY^R5Y6*A%J! zhiF@90KjEaO4ZoZNY`PVKEMon`Kewo(M;cs}C zS9k0eKEZBLF+>U!4lV@vEoTUNRQ{w+NhR$moo5s-L|_LzBZ=bQ_?rK<&56i+5IQVp zENuC! zrNGW*!QpQHFd$hWETpa21}%N8T|L8R5DiQUf?CI1&7Hj2>h$qcP{5WgOEauo32uqV z^fzBM-F8#?!9&H&oFJ$K)hz>5@{&@z4BC{r0Z1|oSs*6~EwhwH6sLadJ9ED zTZ8~qL&bdD`R8}MYrn5; z=|x2l$c*_f;_?0GQ6OgaqHti7l~K!S!1&|6oWD zsMTm&Jg@PsWKWZZh~m)4W~^>6h1I=NFX!lB*cnSR;90jcZ#V@8xsvUg z;zL=rEQLhpgHD)WVyT%8g-hlo0}T?_?^W!jdjugP*mi8a%y93_3)|XR^K^SYJ(=r zSJ8`KesJ>E8K>zG_K^~US`9t_o6O~xZ;|Ml24{OD?90Tcg;s5q=H8=Y57u((8lUJg zeu%*{#9pqn81DpN^DPD(-hr+X`Wvp_v@br84LJABs$p{&Ee2}@9Lh&fLNsLy!0sU9 z#pfz6Jue<(1Ek}kSstgkFl<^b(LJ%9iS6`@(g3l+*MuuKI%Ev1;???MSxZ=9TZL5r z*ml=aK`wsF3Q$v%MSh)(pi@O2TqUffWXx3O{v`4lr%q^-PL3KpJ93`dpTg`Bg?tIU zdd~f2%64B&I%TWZ--S=L1LWr%hY_Yh#~#1=X*@n>f~-yoF|2XeRizcHMjfbUA!SD0 zKIo#xiGy7?w&Z1(MipN8LzSpeY?I7uCyMxpNKf9U=H_T=|EU=KHj ziNv=>V0a!2LHh!J5(>fAD!_^W5HJh!9WEl~ZBu4*Zbwk)XPE)8j^RNUr(HHg1Zy~L z(Dk`YQkUcsQ)=;MeWZNyor?@)yPgSh??aAhqP}00E5U3~58;S?KG}ViMM_UgcT&~(Am2N@yy>cB(4E;POGyqiLOke;^^T%;q zyJSvh9G^V&AgLg!o_dsep-oKJZxxHru)=~y*VC#-fy&P)=468c*XmEKFHW!@gZJEjpHXA;YG&Jn zfVIG-VC!Uus62g;S`;5+YEISX_WTGRA4mw~<$5|+@WbzCJ5QDX%pO&2yz3XW=g;aT zG<;yOM<9E=mq(ljOYhZajsL39!{O%b#GPqq=&fvA?JbYkKad;3D}x|I+~N&8Z@A`U zJ4Lg4^`FezZuiN3Q!gLNEc^O-W8QePSlMRI=Zxi`cj&DzVAS;UN1KsB?4?Xu7G!y# zDXH%Cb<(DlLH(%Q$YWObtHP@8XGquJEXk2SjzZ?OhMXTrNte4;>bW-W0bVW=-8o$$ z;?r>KOVoXGIl>s5UjjPGnFWl(h!x($4DY_jY_cYPu|<7d-RzZU9S`|cVLHn1i-Rqt zZ?pUE+5KMo>Pq|Ug$`Ur&)J6Z#77@)2ZqG@ICXs|2%R#tK1zGR`mz=`#gYYaIH7L$ zx0$DsjJGNx4`>gUYj=@Z#zeyMONGwr-wc#*cU?BqnAquoWf3+4&9}TQl;jY=v9-Bl zlkHK^rj6_^us`bZ$kthCvA8)09MHU4p9i^iT{YPORZ!%g*dkU2<87};<6S2zf(?<$ z2^W;IQ-hC|fV1Pm^Ma5gjs5P-oAG5qn82AAU9{AR4UT5ad(Zj~Q*_e%gvu-v=-zV= z3PP(<=c}=n7Gq(~D;Rbss~6f@RNK--b7ukxs6(MgT;XY#Z;6v}*XjbV9{?pCGqTZF zi>wz$M^1UeX^Jr50jU(C6pki3(U;$b!g(0X;=2e)cG{&YZb#A~ijsqCg(Hp?k!PLW zaE^P#*SLX2=Bt8R`4g|F;)9yET(W2=1hWk9C?w+?TLTgi{eX7c8NFJ?L4T^1mXSX!B1b=Zw^cel$Mgn~I62%) zA{{D*fNx76t{zxWA%-pD)*mYFAnmwt)k-_pZg3<8xPfR2lY#;ccDW{WOpHSJ5cN$= zCWEj_CnLgtY>OZgw?-Bf6BFkj(H-U$RQ{J7*4G~1L8!OsIS~#)@lzv^H%|7*#fBFl z#Fl5}F0QNwBIP-h2FG_Bf?A7KQn)983kFka`VWXMtp}z2ytQO;m)Y3HJyn*+MefhX zKevz8WPDJv4L5DlFzLsVS{gp-17+A)FrItq zKo8r`Md%FPzWWn=D-CK?x<|ke1wd@jq?9Tr?rCllW)+8hm|2@xkj}g8uwmbtJFv1W zZT-C-jW+PlLY2Ac5AM+$R~7z09&G=LF*Kh^r1vJ zYLiCpMJ1~XCVlDbDugy_ZC#6U0DpmF*Lc$Ja4w7by6TW!Y{J5n3>7Tgh#)Ic3#mC* zc~I@f8fyZ41xR?wxH>&O&9GwC=LDV+gu7*@BidG1?!6NZ>-%IJ^gQ5+3qE$&2@c~x$AG6!i&s*N!9;1aei}a z*I~Uvc5I}$EW83DOsS^$I7YO;uS1s}>2ni&BL6I}MAQtq4|*y-?a^Wejn`{=ShjXx krv4AViSz%wuuEh0&38MD!ul(&Dg6EVy8r!|-~KxCCtrP6Q2+n{ literal 0 HcmV?d00001 From 3cc70cd0b78b1fc8be9ad42be6f1ed52af4f4647 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 12:54:16 -0400 Subject: [PATCH 319/417] small changes --- doc/fcn_2D_segm.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt index d03b5cf4..8f9d0663 100644 --- a/doc/fcn_2D_segm.txt +++ b/doc/fcn_2D_segm.txt @@ -1,4 +1,4 @@ -.. _fcn2Dsegm: +.. _fcn_2D_segm: Fully Convolutional Networks (FCN) for 2D segmentation ****************************************************** From 6fed95dc435e8ad3b6560a6915fe9d2047967281 Mon Sep 17 00:00:00 2001 From: Adriana Romero Date: Mon, 1 May 2017 14:12:54 -0400 Subject: [PATCH 320/417] fixed details --- doc/fcn_2D_segm.txt | 83 ++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt index 8f9d0663..0db4b2c9 100644 --- a/doc/fcn_2D_segm.txt +++ b/doc/fcn_2D_segm.txt @@ -6,11 +6,11 @@ Fully Convolutional Networks (FCN) for 2D segmentation Summary +++++++ -Segmentation task is different from classification task because it require predicting +Segmentation task is different from classification task because it requires predicting a class for each pixel of the input image, instead of only 1 class for the whole input. Classification needs to understand *what* is in the input (namely, the context). However, in order to predict what is in the input for each pixel, segmentation needs to recover -*what* is in the input, and *where*. +not only *what* is in the input, but also *where*. .. figure:: images/cat_segmentation.png :align: center @@ -20,24 +20,24 @@ in order to predict what is in the input for each pixel, segmentation needs to r TODO : reference de l'image -The **fully convolutional** network (FCN) owes its name to its architecture that -have only locally connected layers, such as convolution, pooling, upsampling and -no dense layer. It reduce the number of parameters and computation time. To obtain -its segmentation map (output), segmentation networks usually have 2 parts : +**Fully Convolutional Networks** (FCNs) owe their name to their architecture, which is +built only from locally connected layers, such as convolution, pooling and upsampling. +Note that no dense layer is used in this kind of architecture. This reduces the number +of parameters and computation time. To obtain a segmentation map (output), segmentation +networks usually have 2 parts : -* Convolution path : extract semantic/context information -* Deconvolution path : recover spatial information +* Downsampling path : capture semantic/contextual information +* Upsampling path : recover spatial information -The **convolution path** is used to figure out and interpret the context, while the -**deconvolution path** is used to retrieve *where* in the image were detected the things -detected by the convolution path. Furthermore, to fully recover the spatial -information lost in the pooling or downsampling layers, we often use skip connections. +The **downsampling path** is used to extract and interpret the context (*what*), while the +**upsampling path** is used to enable precise localization (*where*). Furthermore, to fully +recover the fine-grained spatial information lost in the pooling or downsampling layers, we +often use skip connections. -A skip connection is a connection that skips a least one layer. Here, it +A skip connection is a connection that bypasses a least one layer. Here, it is often used to transfer local information by concatenating or summing feature -maps from the convolution path -with feature maps from the deconvolution path. It helps combining context -information with spatial information. +maps from the downsampling path with feature maps from the upsampling path. Merging features +from various resolution levels helps combining context information with spatial information. Data @@ -50,10 +50,10 @@ Polyps Model +++++ -The architecture for FCN network depends on the precision desired. The Figures -below show 3 different architectures : FCN32, FCN16 and FCN8. The convolutional -layers are represented as vertical lines between the pooling layers. -Those pooling layers explicitely show the relative size of the feature maps. +There are variants of the FCN architecture, which mainly differ in the spatial precision of +their output. For example, the figures below show the FCN-32, FCN-16 and FCN-8 variants. In the +figures, convolutional layers are represented as vertical lines between pooling layers, which +explicitely show the relative size of the feature maps. .. figure:: images/fcn.png :align: center @@ -61,23 +61,24 @@ Those pooling layers explicitely show the relative size of the feature maps. **Figure 2** : FCN architecture -**Difference between those 3 architectures** +**Difference between the 3 FCN variants** -These 3 different architectures differ in the stride for the last convolution, -and in the skip connections used to obtain their segmentation map, as you can -see in the image below. I will use the name *convolution path* for the network -up to *pool5*. Note that these 3 architectures have the same convolution path, -but their respective deconvolution path differ. +As shown below, these 3 different architectures differ in the stride of the last convolution, +and the skip connections used to obtain the output segmentation maps. We will use the term +*downsampling path* to refer to the network up to *pool5* layer and we will use the term +*upsampling path* to refer to the network composed of all layers after *pool5*. It is worth +noting that the 3 FCN architectures share the same downsampling path, but differ in their +respective upmsapling paths. -1. **FCN-32** : Directly produce the segmentation map from *pool5* by using a -deconvolution layer with stride 32. +1. **FCN-32** : Directly produces the segmentation map from *pool5*, by using a +transposed convolution layer with stride 32. -2. **FCN-16** : Sum the 2x upsampled prediction from *pool5* with *pool4* to further -produce the segmentation map using a deconvolution layer with stride 16. +2. **FCN-16** : Sums the 2x upsampled prediction from *pool5* with *pool4* and then +produces the segmentation map, by using a transposed convolution layer with stride 16. -3. **FCN-8** : Sum the feature map obtained by summing *pool4* with the upsampled -*pool5* with *pool3*, and use a deconvolution with stride 8 on that feature map +3. **FCN-8** : Sums the 4x upsampled *pool5* with the 2x upsampled *pool4* and *pool3*, +and applies a transposed convolution layer with stride 8 on the resulting feature maps to obtain the segmentation map. @@ -87,13 +88,11 @@ to obtain the segmentation map. **Figure 3** : FCN architecture -As explained above, the deconvolution path is different, since it uses different -skip connection layers and different stride for the last convolution. It thus -yield different segmentation, as you can see in Figure 4 below. Combining layers -that have different precision helps retrieving fine and spatial information, as -well as coarse and context information. - - +As explained above, the upsampling paths of the FCN variants are different, since they +use different skip connection layers and strides for the last convolution, yielding +different segmentations, as shown in Figure 4. Combining layers that have different +precision helps retrieving fine-grained spatial information, as well as coarse +contextual information. .. figure:: images/fcn32_16_8.png :align: center @@ -101,8 +100,8 @@ well as coarse and context information. **Figure 4** : FCN results -Note that the FCN-8 architecture was used on the polyps dataset, -because it produces more precise segmentation map. +Note that the FCN-8 architecture was used on the polyps dataset below, +since it produces more precise segmentation map. Metrics @@ -121,7 +120,7 @@ Code - Citations - Contact Code ==== -The FCN8 implementation can be found in the following file: +The FCN-8 implementation can be found in the following file: * `fcn8.py `_ : Defines the model. * `train_fcn8.py `_ : Training loop. From ae93f3af646040e3cfebcfc0702c5d119bf5063e Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 13:20:25 -0400 Subject: [PATCH 321/417] polyps dataset explanation + metrics --- doc/fcn_2D_segm.txt | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt index 0db4b2c9..325f3049 100644 --- a/doc/fcn_2D_segm.txt +++ b/doc/fcn_2D_segm.txt @@ -45,6 +45,13 @@ Data Polyps +The polyps dataset can be found `[here] `__. +In each of the training, validation and test data, the input images are in the +/images directory and the polyps mask (segmentation map) are in /masks2. The +segmentation maps in the *masks2* directory indicate the presence or absence +of polyps for each pixel. The other subdirectories (/masks3 and /masks4) are, +respectively, for a segmentation task with 3 and 4 classes, but will not be +presented here. Model @@ -107,12 +114,35 @@ since it produces more precise segmentation map. Metrics ======= -1. Per pixel accuracy +**Per pixel accuracy** -2. Jaccard (Intersection over Union) +This metric is self explanatory, since it outputs the class prediction accuracy +per pixel. -More structured +.. math:: + :label: jaccard + acc(P, GT) = \frac{|\text{pixels correctly predicted}|}{|\text{total nb of pixels}|} + + +**Jaccard (Intersection over Union)** + +This evaluation metric is often used for image segmentation, since it is more structured. +The jaccard is a per class evaluation metric, which compute the nb of pixels in +the intersection between the +predicted and ground truth segmentation maps for a specified class, divided by the +number of pixels in the union between those two segmentation maps, +also for that specified class. + +.. math:: + :label: jaccard + + jacc(P(class), GT(class)) = \frac{|P(class)\cap GT(class)|}{|P(class)\cup GT(class)|} + +where :math:`P` is the predicted segmentation map and :math: `GT` is the ground +truth segmentation map. Often, a class is well segmented if its respective jaccard +is at least 0.5. In the polyps dataset, the jaccard(polyps) must thus be at +least 0.5. Code - Citations - Contact ++++++++++++++++++++++++++ From ae295bd0d69875498f447311985bd96ed0191807 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 13:33:34 -0400 Subject: [PATCH 322/417] jaccard visualisation --- doc/images/jaccard.png | Bin 0 -> 11118 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/images/jaccard.png diff --git a/doc/images/jaccard.png b/doc/images/jaccard.png new file mode 100644 index 0000000000000000000000000000000000000000..2e7d684708d103a7b4fcbcf362139c915d8d1ddf GIT binary patch literal 11118 zcmdUVcUY6#(r-`{5Ea=-mm(@CJrt2pm8NuQ0qJ0#4xve?Ng&*py|?G=?>^_8yTAL~?~ltv9`X*eX3aZmX04guniu-I8t2Y%oB@GA z=kDEAH2{H5n1Vo5!?dRX%EE!k4Dg2?sb-2aL^vRQY&~G0+Yb?TFy4D`TSu4y%=V$* znd<2CsvtaIywajKMeM_yb|Kzn_}Wp zVqjSjup|Wdl@jOu^M?LV$N@Nfji%F2p~ zONdEGhyWZSo_=mfTVD}3&nrJAsKPw$J)GT<&ImW&V~MtQ2rr}pAE4;B!`zVfR2BGu|A;&vNZG+| ziP%ZP#6=wJ>?I$9VUki(QsAF*{?&MO={u5O$y?GgDw2|5u-Z*oHT7HS;u5!I#ntXe zsY|N<9DC2r6KU&a5BoW{Gcfm`W2OG%Scr-T%od68FhU?6{qzETCj=7V>4b3SRZ%%! zb!ky)-s?KH_Rem{$m zjveMN4)l`*c;)fee}n_@@Q;9kxdBn+0fY;utVcKqBoK8^^|q1k#Ofq{<)m?tz>lD< z6NFHkCktS^78T|9XIhmvU%W22_-2zI6*uudzGJ35@)V131>}^2m(8?9j8QuWjbTsT z_0vQ8H`HwkX)kuL3qSdViMlO^`p!MxM37%vT{F|s(W3BNhnnQfT_M2R5ok^@9$Mfod$FA27N6@n!7N4S` z6%eaL#mUPc@oM=Wt0UV}M;i;sBS~}V(_B82=I8HfToS*W7Wn03*^kFE`8HHB_ow#T z-%41-u4p<>ZFbN&m!u`cTaA~YBuG^nH6K4PT#}^fzHs@IX_dp!E@AgIjrQGc+@U!v zm-ZQRW)P+sp~XWP*us$Y9^tGG$4xIEOZ0m+;TQ;oI9Ut49kMD(gPfY6?-yBFBWXSNBZYuT4J=J++CH+)cV9h zs(qT%XA5XeiJK$Bj*KV7XS7!*6t31*@L3E5_cfJQOmFhrP=QvjonQflhMofP0uQvn z5qbRilk{)M$DBX80nVS?|AGAXOirk(^ShJGXHgoP&fa*y5pT$~*iM-(-X2(20hYWrso`k95>JF< zu&!2PZ*?iSiYxqu@<;gf$SObAYd~0I#KNuJOAZ$DAjpY*JXpYnO1vN~U^sr;Xz%6r z)?#(`ngZ9fq}ax#%(VUDHJo$ec~_n_S=>u`o7J}So~59UyqezBcuJ>vPOi_l2@U&x zFwKUHsHOXA{C3#~0cc!Z*_~7LHfQA)KG7{-DalC9(O%MQUdMs$_!~-sYyMBt>nBK(`3K3J>R99HGnL?y0@I#x~F9ir-pty!8VJNbezJE-X#PGr3)>_rEN|G z4elV)+%zN#+4!jc8}ZI&X0ghyUM6B*{ZSy@x4bdr%VnC}XAFv_t$;tDts-xP@J@Pk z69|<2T#6fnyt?O9$UP;>rrw~A zOpsWS`5v;hQ`q#!0Fv-k;2b_Eyh3iHQK-nmPh)47-7!_sMM;3y2K-btB4n#41d5Bp z4&2j8mp7ZO93adK4Ysp}l)FsNx4n3w=kCTqk*L2I%&_ds^!(S{PBG zx6(G~Ou1YZ^Y}z8NmW!(+A9_!rtY>nrVVqy>PZ(uI`lUj6+O*%WM#gW)^e1#FhecU ztO6fQo7Qie&#!5T9d5Zea;?|S@}N}oE2j>KA-8^WXC_}2g!}S$uCr`egRavES>St4gTzO+|42fe1p@D{d!eM%Hyy(WHq> z;M_$5c4uuQ(T@G;1+vq7Eoas1fSKvL4|%%fQhjiB%j1qRA2W^)GR$$s2W-AWDY!B5 z?Q;)97`DgoZJWPtbYF>+&mtw|9ZUw3yV6G+bNTmKI-g5Ewtci*U?j*HfYuwmXm5^w zs9gB^B2y;4R$_1kgsy%`xVUK-Z)Lqm=w^MJW{gBKelM=am+UTmiqb8unHr7WYnq8_ zb{f0g?ka4)d+5PQmQ0ddNVYtandFpC^F+N{LTXT3IkNX4-cVO)!`f$3`q=}Nu;OT= zpX1xTW)ep1;}kJ%))hB9VvbmsN$o)2TCY7a!iDTl9~oBkovvs$m#rpL7b8-%kiIX=wNR+#6>L*7A)2nBSL;e9s=Qxi5;+-@>`up4(Q1kBY2LIUo2~E z$mS*FA`Cuf;j6K3VhD{>m-nh`nyMPP(#*7j5#G{2(2${};l)0sW(Vgdd2N>-x7p4I z%x^S2p3T9hW|CfEmK5x)2A@5bx9qTsYVvT$3S;Mr|;*@HL(>6DQG`8p)yU?p}dJkiytMY>9-^sR)DnwX3dD zy2K#qmMW%>*`v*rFH&CkVHf7h8SlBczN@hwn(5GRCzSifCi!y3MZ;ry!6on8f9T`t z)T*X~HptHuAoC=ztOu6feAP9ImsIDog2x2HUluo0qG16YrP$T+ih?!f*6{nM1Y0!0 zS6N#PrR@XB>qSxG_;+TIn56J(f~2~F&eK$Vf9Tt#ANt3u?6R|a%m|V z4Qpd^ZlS=c30LpvMQR#sdL3+b?hrXvMoW0~Ig1|bxa;f)PQ67#hT}pG8VSLQo$!xe zBgD4en~Cvof6Ex^J)E2>!(`}kiS=A!+N|3;iWn(0TjAloP|8+Bxco-`f|NmFEXF(+ zT2eFJk(*&P^+TFWFH9`OGU>^ic|T}}B8*>;_1fw7D9>RzvHh$0yTD<2$FBi%`Rnx6 ziJJ?uVYqO*t?AFckGQiY>YB-N)yYKRHASljviyWTo^BP;hs1m9_NL$HyCSq_9w37( z1+Vo+q8J>5&JE0M#dr|)4Zi8QI4nQ)FFSqzgXUfBvAC*~n+Tl6+9!e}`Ly5^j>H5a zt$gzYC+1l)rP!m5$~I;KthF4^&UudXtwi0MQ&J0dKL*ojzh4kc7e>--Rn}eMd5K@tVt%tk zbEQSicKy4obaTL7L*Mn$SQq3?TSfV#zR{KV&zH>YH+`w7g~R&^^EK!3#LYyC4N8It zxm$E9#xi$yG((psi}VZCbdvqcVnw%(7rKvF2Ak)N$%-Nqy2?ef-}dXgGT`Tt+F}g@E@JU1IWM6EX>)p%zwuv9E<@8>HJ> zNE;d4+R|E7jW7-=o1x5MUe_ukDyY`9tVt>OouwHeMHOo%6es(V#5tn71z30@hNLEf z6?uF53yQbQ7lyKxX%n0!z8bGvq}*@qY=7rbh7T;8`fw(hfnmg&BX<%oDa5O>(nEXB zwlLxA=5eUK67wG)oq#so2+utOmd8W|>J%;IrLdD;oEPSw?$kH5m@2|_sUYz{BIB#P zW_;v#<<+)8i-uxIE`{`R?Sk!sXGTMP&WEGzTrv2KhiZ{uTfK#Ge=J0GV{Pd=CCLTlR}!#zcty!yqIxRM(|Fp zo*E@wxZm_eMW?LGM^Q#l(Gc;OFPQtYy?D2@h4*;7bjDt?t0H!VtX5mUJs38BNXwd5 zUyFD?n^j=+$UJa_A}BbJv&l}7)UcN$n>R}0N@!ES+NNh2c3Fq9Wz96GH=SD4pJ`D| zO%B?{eTlh|E>rpu>NZaA<`g?rJ5ziv5#6nY@JA#xc|D_Asxy!Fv>FXhnjX@h@q3iv zkA{W@6w0#m;C$+byT}R1h-WHhaLfWKubl`H%0nK@C5rBVlA7S zHqqek%v4pT`cgQt0!hDlL7eAwOSfFTUi3AlD0uWMW^2YIJKOHSd(V;{a&5gQ6c{rW z&-nZG2q&2pzF>x%HR^`kZsPi51uw=$_pzvi##%`-lyB}Il*^v|hLRJSUvp-{oN;P? zw#zm8!3T1zTMqL?x3O0b%P?@GBg~VA>1@_ez>m*3*36if&I4Jl-4`wxX9gB! zq0_A`VtgnETiv>zpP$mB`Vih_ouswc(HkS-_!R}LLS0i0Ji1+GHaVnyQRt~jh@Gjp zAhf4gJ-y@wMpK_>^$(|J6a{k7bQg1(t#>nC*qwJBLxt)(xjJC2Ror%d8F5eHbSRxN8R_Fmb+ZKHs!C>Bg6I=nUt zE<6DfpN14@_k@4>yuR@nNCtm#yeDcEsNyvy$xs_><&zipVqi69bUh`Qv^_F~d@kpY zwi>yP-1c=fhGrX#k8K6jFW4iZ{VbXf=Ea5ys(4&ZEB#JTi2ru*gO=n%rBGzb*8P9Dx@vQ~WB%$^0s z4eQCtRZVtmiq;!03*Xo=^jh>LYgKSZ&FEmXY|!TPdZp!N)cFV$=aip~%j2v5Z$eM? zXQ5B9s5q{?)FhF_sy|{g{I50DR%P9GLeVKa!KYR1-P?*wPIb`bp282wu=7#ds3uFH zg+zQ?SL>BR&_~JoAY8J*^Ngp^bVF^ zrS_Zx^&5QZ8r@r-gK^spx>Br8(wT}iZ5#BsZhI3MK%w4}sjg(4C}c*8oz1b`F9V#T`F zJjFA7d1W&J!OTQ9NF#+LmLs>m--$m|i$=e4=u6N3^%N+E1D-O6e)v_!r3F(|gR?lN z<5zp8lbak~>AQIjW-O$7&9ycB-0o?RxL{@WQ)Ub*#f@a;rz8kv{7CX>XAXJ2-n92b zB7V}dFQ_#qirKijq8H-WU&3_RgEx^#1yZlBHYznK@~_PSocQVCi>{GQrC=++BOueR zkpYF?&lF)U1d<19+{*%;!%`cBuKeb6f;U*e_13MqSIM@yhlyH`aJB59)K?P7L^dyVQ>>5Jp%Hc|Lu@-qlNo4K2*#Ix7UtP{; zy|7nEY5KWL4y+q}%_f!^TMb7Ahita@aGV4|99_%1x}XpYC_Si1moW zy5g;PM)5GOkrL~vVb{2;Oai)v!O+hqLU)A(P2ceAaBx$t0H?JmqO*!7#q!-@W`qgm zdxqqq7iL3)+pmo~E?~4_p{h~`1ub#jpXj+Rx+iJc$>sAXS?TBs9J{GR{Hm#;1d0K{ z=}Zy+L!8~%$2U?P1W+YFo$nX$`DxtSnw1RBIM-`7n3C>}4A4X4umfq{Y=}o7P&){#NSxf5$6S_LmH-@-Qfb6>u z`yS~m9&YF?6QQSsAlEg`@_$v_TR1s*44~&8>;mw6J^;U$Eh^RTDZO&5gZcR#4I75Y zqy_B;NNGn*kcu0EQ_&<$33Q_`T|v^)Z@y)_*r#V@ukLt(9@G(D5-3%pR+9v>(oYnL zKoL;f%ysTf9j!X_T)_Hwu;(R`MHn~pBQ)oznR&RYxXIq5uc-FrjICq}fynvAwM7SqWZ zT*N|RwYrU?uPM33J63t|kiS)_&M$yNCVXL^&UARJ)KnvHSj?Lnmy6O$G$p96a@^!YWJ5DL#rk~_2KK6l8$YkVRx z5J(4;2w$!o{mLEaZoIR*oZEbbEAYqXWfdO(5F)u@gOaa)0z?&UaYX0z+t9U8^$6;je+#ipU{62#uqZjPjH=-+k zBV@kcC!S2?Eud1);GYJsSY8|y-7|!Vpd7I4-43z)^O>K zp$BKI_Ny@H+MU1Lzq$e^S7qYg<`N9EA3>f$L-cqtF*Jy$6HZd>|q_gAk3L^5O+2& zEz0Wy35%nRNVFj2CzlR8nT4z4TOB3Ig$aw8hc)UOmfv+>akRj=W2Wiv768Cnbq&8LQT(m%#{a<-yf#u+s(nV_K* zG^dQ4%NXNL|F|V726^|V*_(SBGv#wVM+-zJst%?tsR^+BSPW{X+9S z#PXYrMjN9`S2l@TrGr>>l;A+FQjqh5#{KZ7iR!fPR-qJH0J+JsTe@0sa)16hfr~(yxFdXi8y@|1ZbzjaoFN zo>V*5L2KxDN+kR_p;w~7a3f%fXO*#K<}uXD&x;sbBr3;EKmJz;2;SKzqN5QRJPoPncE86D>S!y_P&`g?6Dvw~prYdQo)a_bn~T=Q!sbqxyJI z!<`ql^efl&-Nck7{|jDoE}_lfL|EXB2(+rWAfy}C?N)HUTaAY0vfx77MJ^avb;!v| z7n!b=h%*I}99u!i#GT;$q@}yF$8gQ_d^VFV6iZQL?=1!pW4s*}KLwO+Mt7pRv?5)! zgoQb{0uL0?d~6qIyf?SJzcH-DJ5|s=0|#1Ie~$5vbTOdRWd!>lzN$LbD>PKi5cBMh z%tXmmoE4LJ$EbX!H5pw`wj1 zd*PlDC~MFtQnHc6RKnWowZoS>ukx(N2p(0zp<>Y`09nYIxF5(Ba_~3^-9g$piPA4T zhFrp~O})jb+5lTapU}pl=Xdo2wwX1(vq$F>tJm6gfg)b&8joGKT#u#$}LxvV(O0cxI%_we0PMM-FY2JE!N>^=>!gA zv|S6OFND$uY|1^Dol*h-uKtOl7&HfgeIDKFE1Te~6ag+D@ZSM|IGOyv10x6PK8ePQ zgzYNa{oqX(;6i07a=shEHN5pfLTaq*Hu~m$p&T8ov`%t{X%cro0qZ+OZZub zZ%OwjJH||ads?dKmoHu{5c|!_6b&T_?VR=sxh5w^9(Qt?pH4HS*uGB;f6-;kU7U3X z)5Vw9ldKrD7j_;WagU5ee~-Um2mz|ZY~Ba=l{TA)u(u4bEboo^>*DSM5MNoak)l~+ zb4j2?;MwjSneiqZfV6dCfvXJvPZ^hW`}&~EFHhaYauwe(YT>9d58?LD^InGv%M9r5 zsFdz(jK*M4qevvDh6@&0qe@{4DXs!U|4$s^le5?1djJxs-S_+nLsTss!w?iS@lIaT zF%a=bme*eDB#CFe4X+_}*;0ag@~oc~F#WEn@~12qu_Y$xxW=Culntw0)fOU1SMj}< z21?P{*n9inO14p!M^y&Qwlm?JEDU)~cqN2IakdheI;C%~pc!r7(55*b=G%al9U6R92 z)~HHZ?3F4w)#=u&dtiRS+kX9DBFGP3Gi-$c=b^LKmQ9um3ntOdIg<7?Z*yM9JSM@nFN|JnV~i4`iA#WrYSXh=m@<`ni)21F0i-+Z2zNCq z%&ra`uq|!hd^ucw>O!g0gVdeGqaCIGubTMH|4U!o*UgjTIN!9!GsD;bZ~qhCm8~_+ zO&=eO-%!LBS1&c&u|AQWKqB;h;VFjGi|dEecD`cs{>sV(-<6PVBK}4*1r$8z!Gd>A zE}LGtG`Ho>;gxfLdViV$x;l{A5f(E0bTH_}BF>|`2Gz&4^vdaQcVQIv2o@lBtw4D* z_;J}BdP0K1WOEESfNws!)8!5~4}G%qk2{)GI5=(vxb=fgG}bj7n4R5|uIM#+-dXGJ z9t>~rUH@L^xm%02{5E@on^h(4q~*lrh46L{svRtLNg?9FBWy*VTb`A{bs5iv8046( zr(AieT0>1pZS|UyM@La#c0~GA=H;bA&1;l5Il8Nw5o=19i{w|{A7`6Da(se?))amo zNdBlm6Waf|E#)s=6z^O4olbz>4^=-Z`G`rW&NW1gi%Xm^fNE}}HtCVZ2Hv1p^I1wk zp|*~W*RqS`1eag1*2E>jt;j9A+eKC8gK@o(>pgJrL!g)kEvfBQ|6cf49>@c=_IuCz z`huyBoxmxdKuryuj9Yv+$wa7kjJx`(4kia844k}37ktN7}SOdU!xF>5x4(*0szCOHNA3wtuY;@dbVCbl= zd9;;AlQVSG`ccZ(Li-CxzP0!PP!Y%2YH*8R4*D(ftMg!O@%G9(*OeT$%H6QF<@vRS z7URUEu+USV+_%zY$tKO}zfPHNT0Gb}sK&2t&KgXhhk_saf&o|gH!TMODs*+x230JS5xu;f@V1Y#;*fvx_JHXi@IOtKSoxH9ZhCzd*v=F{D+fe$$!V@pg1 zcNn~K;c^M3OQ}Z(^+BlL1?0)tT4-iKf;AyF2GIJ$eCS}bdC%9Z!_F}|$Bj1&JkoXk z4aSxUnlSsmz2)RzK<`CvzIz5>V-7%&{2yFC=EURtRQOkEI$* Date: Mon, 1 May 2017 13:56:52 -0400 Subject: [PATCH 323/417] code for fcn8, miss dataset_loaders --- code/fcn_2D_segm/__init__.py | 0 code/fcn_2D_segm/data_loader.py | 125 ++++++++++++ code/fcn_2D_segm/fcn8.py | 200 +++++++++++++++++++ code/fcn_2D_segm/metrics.py | 135 +++++++++++++ code/fcn_2D_segm/model_helpers.py | 109 ++++++++++ code/fcn_2D_segm/test_fcn8.py | 163 +++++++++++++++ code/fcn_2D_segm/train_fcn8.py | 320 ++++++++++++++++++++++++++++++ 7 files changed, 1052 insertions(+) create mode 100644 code/fcn_2D_segm/__init__.py create mode 100644 code/fcn_2D_segm/data_loader.py create mode 100644 code/fcn_2D_segm/fcn8.py create mode 100644 code/fcn_2D_segm/metrics.py create mode 100644 code/fcn_2D_segm/model_helpers.py create mode 100644 code/fcn_2D_segm/test_fcn8.py create mode 100644 code/fcn_2D_segm/train_fcn8.py diff --git a/code/fcn_2D_segm/__init__.py b/code/fcn_2D_segm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/code/fcn_2D_segm/data_loader.py b/code/fcn_2D_segm/data_loader.py new file mode 100644 index 00000000..72136458 --- /dev/null +++ b/code/fcn_2D_segm/data_loader.py @@ -0,0 +1,125 @@ +from dataset_loaders.images.polyps912 import Polyps912Dataset +from dataset_loaders.images.camvid import CamvidDataset +from dataset_loaders.images.polyps912 import Polyps912Dataset +from dataset_loaders.images.isbi_em_stacks import IsbiEmStacksDataset + + +def load_data(dataset, train_data_augm_kwargs={}, one_hot=False, + batch_size=[10, 10, 10], shuffle_train=True, return_0_255=False, + which_set='all'): + + assert which_set in ['all', 'train', 'val', 'test'] + + # Build dataset iterator + if dataset == 'polyps912': + train_iter = Polyps912Dataset(which_set='train', + batch_size=batch_size[0], + seq_per_subset=0, + seq_length=0, + data_augm_kwargs=train_data_augm_kwargs, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=False, + shuffle_at_each_epoch=shuffle_train, + return_list=True, + return_0_255=return_0_255) + val_iter = Polyps912Dataset(which_set='val', + batch_size=batch_size[1], + seq_per_subset=0, + seq_length=0, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=False, + shuffle_at_each_epoch=False, + return_list=True, + return_0_255=return_0_255) + test_iter = Polyps912Dataset(which_set='test', + batch_size=batch_size[2], + seq_per_subset=0, + seq_length=0, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=False, + shuffle_at_each_epoch=False, + return_list=True, + return_0_255=return_0_255) + elif dataset == 'camvid': + train_iter = CamvidDataset(which_set='train', + batch_size=batch_size[0], + seq_per_subset=0, + seq_length=0, + data_augm_kwargs=train_data_augm_kwargs, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=True, + shuffle_at_each_epoch=shuffle_train, + return_list=True, + return_0_255=return_0_255) + val_iter = CamvidDataset(which_set='val', + batch_size=batch_size[1], + seq_per_subset=0, + seq_length=0, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=True, + shuffle_at_each_epoch=False, + return_list=True, + return_0_255=return_0_255) + test_iter = CamvidDataset(which_set='test', + batch_size=batch_size[2], + seq_per_subset=0, + seq_length=0, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=True, + shuffle_at_each_epoch=False, + return_list=True, + return_0_255=return_0_255) + elif dataset == 'em': + train_iter = IsbiEmStacksDataset(which_set='train', + start=0, + end=25, + batch_size=batch_size[0], + seq_per_subset=0, + seq_length=0, + data_augm_kwargs=train_data_augm_kwargs, + return_one_hot=one_hot, + return_01c=False, + overlap=0, + use_threads=True, + shuffle_at_each_epoch=shuffle_train, + return_list=True, + return_0_255=return_0_255) + + val_iter = IsbiEmStacksDataset(which_set='train', + batch_size=batch_size[1], + seq_per_subset=0, + seq_length=0, + return_one_hot=one_hot, + return_01c=False, + use_threads=True, + shuffle_at_each_epoch=False, + start=25, + end=30, + return_list=True, + return_0_255=return_0_255) + test_iter = None + else: + raise NotImplementedError + + if which_set == 'train': + ret = train_iter + elif which_set == 'val': + ret = val_iter + elif which_set == 'test': + ret = test_iter + else: + ret = [train_iter, val_iter, test_iter] + + return ret diff --git a/code/fcn_2D_segm/fcn8.py b/code/fcn_2D_segm/fcn8.py new file mode 100644 index 00000000..a5f6aabc --- /dev/null +++ b/code/fcn_2D_segm/fcn8.py @@ -0,0 +1,200 @@ +import numpy as np +import scipy.io as sio +import theano.tensor as T +import lasagne +from lasagne.layers import InputLayer, DropoutLayer, ReshapeLayer,\ + DimshuffleLayer +from lasagne.layers import Pool2DLayer as PoolLayer +from lasagne.layers import Conv2DLayer as ConvLayer +from lasagne.layers import ElemwiseSumLayer, ElemwiseMergeLayer +from lasagne.layers import Deconv2DLayer as DeconvLayer +from lasagne.nonlinearities import softmax, linear + +import model_helpers + + +def buildFCN8(nb_in_channels, input_var, + path_weights='/Tmp/romerosa/itinf/models/' + + 'camvid/new_fcn8_model_best.npz', + n_classes=21, load_weights=True, + void_labels=[], trainable=False, + layer=['probs_dimshuffle'], pascal=False, + temperature=1.0, dropout=0.5): + ''' + Build fcn8 model + ''' + + net = {} + + # Contracting path + net['input'] = InputLayer((None, nb_in_channels, None, None), + input_var) + + # pool 1 + net['conv1_1'] = ConvLayer( + net['input'], 64, 3, pad=100, flip_filters=False) + net['conv1_2'] = ConvLayer( + net['conv1_1'], 64, 3, pad='same', flip_filters=False) + net['pool1'] = PoolLayer(net['conv1_2'], 2) + + # pool 2 + net['conv2_1'] = ConvLayer( + net['pool1'], 128, 3, pad='same', flip_filters=False) + net['conv2_2'] = ConvLayer( + net['conv2_1'], 128, 3, pad='same', flip_filters=False) + net['pool2'] = PoolLayer(net['conv2_2'], 2) + + # pool 3 + net['conv3_1'] = ConvLayer( + net['pool2'], 256, 3, pad='same', flip_filters=False) + net['conv3_2'] = ConvLayer( + net['conv3_1'], 256, 3, pad='same', flip_filters=False) + net['conv3_3'] = ConvLayer( + net['conv3_2'], 256, 3, pad='same', flip_filters=False) + net['pool3'] = PoolLayer(net['conv3_3'], 2) + + # pool 4 + net['conv4_1'] = ConvLayer( + net['pool3'], 512, 3, pad='same', flip_filters=False) + net['conv4_2'] = ConvLayer( + net['conv4_1'], 512, 3, pad='same', flip_filters=False) + net['conv4_3'] = ConvLayer( + net['conv4_2'], 512, 3, pad='same', flip_filters=False) + net['pool4'] = PoolLayer(net['conv4_3'], 2) + + # pool 5 + net['conv5_1'] = ConvLayer( + net['pool4'], 512, 3, pad='same', flip_filters=False) + net['conv5_2'] = ConvLayer( + net['conv5_1'], 512, 3, pad='same', flip_filters=False) + net['conv5_3'] = ConvLayer( + net['conv5_2'], 512, 3, pad='same', flip_filters=False) + net['pool5'] = PoolLayer(net['conv5_3'], 2) + + # fc6 + net['fc6'] = ConvLayer( + net['pool5'], 4096, 7, pad='valid', flip_filters=False) + net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout) + + # fc7 + net['fc7'] = ConvLayer( + net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False) + net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout) + + net['score_fr'] = ConvLayer( + net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False) + + # Upsampling path + + # Unpool + net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2, + crop='valid', nonlinearity=linear) + net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1, + pad='same') + net['score_fused'] = ElemwiseSumLayer((net['score2'], + net['score_pool4']), + cropping=[None, None, 'center', + 'center']) + + # Unpool + net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4, + stride=2, crop='valid', nonlinearity=linear) + net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1, + pad='valid') + net['score_final'] = ElemwiseSumLayer((net['score4'], + net['score_pool3']), + cropping=[None, None, 'center', + 'center']) + # Unpool + net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16, + stride=8, crop='valid', nonlinearity=linear) + upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1] + net['input_tmp'] = InputLayer((None, upsample_shape, None, + None), input_var) + + net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']), + merge_function=lambda input, deconv: + deconv, + cropping=[None, None, 'center', + 'center']) + + # Final dimshuffle, reshape and softmax + net['final_dimshuffle'] = \ + lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1)) + laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape + net['final_reshape'] = \ + lasagne.layers.ReshapeLayer(net['final_dimshuffle'], + (T.prod(laySize[0:3]), + laySize[3])) + net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'], + nonlinearity=softmax) + + # Load weights + if load_weights: + if pascal: + path_weights = '/data/lisatmp4/erraqabi/data/att-segm/' + \ + 'pre_trained_weights/pascal-fcn8s-tvg-dag.mat' + if 'tvg' in path_weights: + str_filter = 'f' + str_bias = 'b' + else: + str_filter = '_filter' + str_bias = '_bias' + + W = sio.loadmat(path_weights) + + # Load the parameter values into the net + num_params = W.get('params').shape[1] + for i in range(num_params): + # Get layer name from the saved model + name = str(W.get('params')[0][i][0])[3:-2] + # Get parameter value + param_value = W.get('params')[0][i][1] + + # Load weights + if name.endswith(str_filter): + raw_name = name[:-len(str_filter)] + if 'score' not in raw_name and \ + 'upsample' not in raw_name and \ + 'final' not in raw_name and \ + 'probs' not in raw_name: + + # print 'Initializing layer ' + raw_name + param_value = param_value.T + param_value = np.swapaxes(param_value, 2, 3) + net[raw_name].W.set_value(param_value) + + # Load bias terms + if name.endswith(str_bias): + raw_name = name[:-len(str_bias)] + if 'score' not in raw_name and \ + 'upsample' not in raw_name and \ + 'final' not in raw_name and \ + 'probs' not in raw_name: + + param_value = np.squeeze(param_value) + net[raw_name].b.set_value(param_value) + else: + with np.load(path_weights) as f: + param_values = [f['arr_%d' % i] for i in range(len(f.files))] + lasagne.layers.set_all_param_values(net['probs'], param_values) + + # Do not train + if not trainable: + model_helpers.freezeParameters(net['probs']) + + # Go back to 4D + net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1], + laySize[2], n_classes)) + + net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'], + (0, 3, 1, 2)) + + # Apply temperature + if load_weights: + soft_value = net['upsample'].W.get_value() / temperature + net['upsample'].W.set_value(soft_value) + soft_value = net['upsample'].b.get_value() / temperature + net['upsample'].b.set_value(soft_value) + + return [net[el] for el in layer] diff --git a/code/fcn_2D_segm/metrics.py b/code/fcn_2D_segm/metrics.py new file mode 100644 index 00000000..715d9e19 --- /dev/null +++ b/code/fcn_2D_segm/metrics.py @@ -0,0 +1,135 @@ +import theano.tensor as T +import numpy as np +from theano import config + +from lasagne.objectives import squared_error as squared_error_lasagne + +_FLOATX = config.floatX +_EPSILON = 10e-8 + + +def jaccard(y_pred, y_true, n_classes, one_hot=False): + + assert (y_pred.ndim == 2) or (y_pred.ndim == 1) + + # y_pred to indices + if y_pred.ndim == 2: + y_pred = T.argmax(y_pred, axis=1) + + if one_hot: + y_true = T.argmax(y_true, axis=1) + + # Compute confusion matrix + cm = T.zeros((n_classes, n_classes)) + for i in range(n_classes): + for j in range(n_classes): + cm = T.set_subtensor( + cm[i, j], T.sum(T.eq(y_pred, i) * T.eq(y_true, j))) + + # Compute Jaccard Index + TP_perclass = T.cast(cm.diagonal(), _FLOATX) + FP_perclass = cm.sum(1) - TP_perclass + FN_perclass = cm.sum(0) - TP_perclass + + num = TP_perclass + denom = TP_perclass + FP_perclass + FN_perclass + + return T.stack([num, denom], axis=0) + + +def accuracy(y_pred, y_true, void_labels, one_hot=False): + + assert (y_pred.ndim == 2) or (y_pred.ndim == 1) + + # y_pred to indices + if y_pred.ndim == 2: + y_pred = T.argmax(y_pred, axis=1) + + if one_hot: + y_true = T.argmax(y_true, axis=1) + + # Compute accuracy + acc = T.eq(y_pred, y_true).astype(_FLOATX) + + # Create mask + mask = T.ones_like(y_true, dtype=_FLOATX) + for el in void_labels: + indices = T.eq(y_true, el).nonzero() + if any(indices): + mask = T.set_subtensor(mask[indices], 0.) + + # Apply mask + acc *= mask + acc = T.sum(acc) / T.sum(mask) + + return acc + + +def crossentropy(y_pred, y_true, void_labels, one_hot=False): + # Clip predictions + y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON) + + if one_hot: + y_true = T.argmax(y_true, axis=1) + + # Create mask + mask = T.ones_like(y_true, dtype=_FLOATX) + for el in void_labels: + mask = T.set_subtensor(mask[T.eq(y_true, el).nonzero()], 0.) + + # Modify y_true temporarily + y_true_tmp = y_true * mask + y_true_tmp = y_true_tmp.astype('int32') + + # Compute cross-entropy + loss = T.nnet.categorical_crossentropy(y_pred, y_true_tmp) + + # Compute masked mean loss + loss *= mask + loss = T.sum(loss) / T.sum(mask) + + return loss + + +def binary_crossentropy(y_pred, y_true): + # Clip predictions to avoid numerical instability + y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON) + + loss = T.nnet.binary_crossentropy(y_pred, y_true) + + return loss.mean() + + +def entropy(y_pred): + # Clip predictions to avoid numerical instability + y_pred = T.clip(y_pred, _EPSILON, 1.0 - _EPSILON) + + ent = - T.sum(y_pred * T.log(y_pred), axis=1) + + return ent.mean() + + +def squared_error_h(y_pred, y_true): + + coef = np.linspace(_EPSILON, 1, len(y_pred)+1)[:-1] + + error_list = [((a_i - b_i)**2).mean() for + a_i, b_i in zip(y_pred, y_true)] + error_list = error_list * coef + + return sum(error_list) + + +def squared_error(y_pred, y_true, void): + + if isinstance(void, int): + loss_aux = squared_error_lasagne(y_pred, y_true[:, :void, :, :]).mean(axis=1) + mask = y_true[:, :void, :, :].sum(axis=1) + else: # assumes b,c,0,1 + loss_aux = squared_error_lasagne(y_pred, y_true).mean(axis=1) + mask = T.neq(y_true.sum(1), sum(void)) + + loss_aux = loss_aux * mask + loss = loss_aux.sum()/mask.sum() + + return loss diff --git a/code/fcn_2D_segm/model_helpers.py b/code/fcn_2D_segm/model_helpers.py new file mode 100644 index 00000000..8a31d22a --- /dev/null +++ b/code/fcn_2D_segm/model_helpers.py @@ -0,0 +1,109 @@ +import theano +import lasagne + +from lasagne.layers import InputLayer +from lasagne.layers import ConcatLayer + + +def freezeParameters(net, single=True): + """ + Freeze parameters of a layer or a network so that they are not trainable + anymore + + Parameters + ---------- + net: a network layer + single: whether to freeze a single layer of all of the layers below as well + """ + all_layers = lasagne.layers.get_all_layers(net) + + if single: + all_layers = [all_layers[-1]] + + for layer in all_layers: + layer_params = layer.get_params() + for p in layer_params: + try: + layer.params[p].remove('trainable') + except KeyError: + pass + + +def unfreezeParameters(net, single=True): + """ + Unfreeze parameters of a layer or a network so that they become trainable + again + + Parameters + ---------- + net: a network layer + single: whether to freeze a single layer of all of the layers below as well + """ + all_layers = lasagne.layers.get_all_layers(net) + + if single: + all_layers = [all_layers[-1]] + + for layer in all_layers: + layer_params = layer.get_params() + for p in layer_params: + try: + layer.params[p].add('trainable') + except KeyError: + pass + + +def softmax4D(x): + """ + Softmax activation function for a 4D tensor of shape (b, c, 0, 1) + + Parameters + ---------- + net: x - 4d tensor with shape (b, c, 0, 1) + """ + # Compute softmax activation + stable_x = x - theano.gradient.zero_grad(x.max(1, keepdims=True)) + exp_x = stable_x.exp() + softmax_x = exp_x / exp_x.sum(1)[:, None, :, :] + + return softmax_x + + +def concatenate(net, in_layer, concat_h, concat_vars, pos): + """ + Auxiliary function that checks whether we should concatenate the output of + a layer `in_layer` of a network `net` to some a tensor in `concat_vars` + + Parameters + ---------- + net: dictionary containing layers of a network + in_layer: name of a layer in net + concat_h: list of layers to concatenate + concat_vars: list of variables (tensors) to concatenate + pos: position in lists `concat_h` and `concat_vars` we want to check + """ + if pos < len(concat_h) and concat_h[pos] == 'input': + concat_h[pos] = in_layer + + # if this is the layer we want to concatenate, create an InputLayer with the + # tensor we want to concatenate and a ConcatLayer that does the job afterwards + if in_layer in concat_h: + net[in_layer + '_h'] = InputLayer((None, net[in_layer].input_shape[1] if + (concat_h[pos] != 'noisy_input' and + concat_h[pos] != 'input') + else 3, None, None), concat_vars[pos]) + net[in_layer + '_concat'] = ConcatLayer((net[in_layer + '_h'], + net[in_layer]), axis=1, cropping=None) + pos += 1 + out = in_layer + '_concat' + + laySize = net[out].output_shape + n_cl = laySize[1] + print('Number of feature maps (concat):', n_cl) + else: + out = in_layer + + if concat_h and pos <= len(concat_h) and concat_h[pos-1] == 'noisy_input': + concat_h[pos-1] = 'input' + + return pos, out diff --git a/code/fcn_2D_segm/test_fcn8.py b/code/fcn_2D_segm/test_fcn8.py new file mode 100644 index 00000000..60a19ef4 --- /dev/null +++ b/code/fcn_2D_segm/test_fcn8.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +import argparse +from getpass import getuser + +import numpy as np + +import theano +import theano.tensor as T +from theano import config +import lasagne + +from fcn8 import buildFCN8 +from data_loader import load_data + +from metrics import jaccard, accuracy + +_FLOATX = config.floatX +if getuser() == 'romerosa': + SAVEPATH = '/Tmp/romerosa/itinf/models/' + LOADPATH = '/data/lisatmp4/romerosa/itinf/models/' + WEIGHTS_PATH = '/data/lisatmp4/romerosa/itinf/models/' +elif getuser() == 'jegousim': + SAVEPATH = '/Tmp/romerosa/itinf/models/' + LOADPATH = '/data/lisatmp4/romerosa/itinf/models/' + WEIGHTS_PATH = '/data/lisatmp4/romerosa/itinf/models/' +elif getuser() == 'michal': + SAVEPATH = '/home/michal/Experiments/iter_inf/' + LOADPATH = SAVEPATH + WEIGHTS_PATH = '/home/michal/model_earlyjacc.npz' +else: + raise ValueError('Unknown user : {}'.format(getuser())) + + +def test(dataset, which_set='test', data_aug=False, + savepath=None, loadpath=None, test_from_0_255=False): + + # + # Define symbolic variables + # + input_x_var = T.tensor4('input_var') + target_var_3D = T.itensor3('target_var_4D') + + # + # Build dataset iterator + # + if which_set == 'train': + test_iter, _, _ = load_data(dataset, one_hot=False, + batch_size=[10, 10, 10], + return_0_255=test_from_0_255) + elif which_set == 'valid': + _, test_iter, _ = load_data(dataset, one_hot=False, + batch_size=[10, 10, 10], + return_0_255=test_from_0_255) + if which_set == 'test': + _, _, test_iter = load_data(dataset, one_hot=False, + batch_size=[10, 10, 10], + return_0_255=test_from_0_255) + + colors = test_iter.cmap + n_batches_test = test_iter.nbatches + n_classes = test_iter.non_void_nclasses + void_labels = test_iter.void_labels + nb_in_channels = test_iter.data_shape[0] + void = n_classes if any(void_labels) else n_classes+1 + + # + # Prepare load/save directories + # + exp_name = 'fcn8' + + # + # Build networks + # + print 'Building networks' + # Build FCN8 with pre-trained weights up to layer_h + prediction + fcn = buildFCN8(nb_in_channels, input_var=input_x_var, + n_classes=n_classes, + void_labels=void_labels, + trainable=False, load_weights=True, + layer=['probs_dimshuffle'], + pascal=(dataset == 'pascal'), + path_weights=WEIGHTS_PATH+dataset+'/fcn8_model.npz') + + # + # Define and compile theano functions + # + print "Defining and compiling test functions" + test_prediction = lasagne.layers.get_output(fcn, deterministic=True)[0] + + test_prediction_dimshuffle = test_prediction.dimshuffle((0, 2, 3, 1)) + sh = test_prediction_dimshuffle.shape + test_prediction_2D = \ + test_prediction_dimshuffle.reshape((T.prod(sh[:3]), sh[3])) + + # Reshape iterative inference output to b01,c + target_var_2D = target_var_3D.flatten() + + test_acc = accuracy(test_prediction_2D, target_var_2D, void_labels) + test_jacc = jaccard(test_prediction_2D, target_var_2D, n_classes) + + val_fn = theano.function([input_x_var, target_var_3D], [test_acc, + test_jacc]) + pred_fcn_fn = theano.function([input_x_var], test_prediction) + + # Iterate over test and compute metrics + print "Testing" + acc_test_tot = 0 + jacc_num_test_tot = np.zeros((1, n_classes)) + jacc_denom_test_tot = np.zeros((1, n_classes)) + for i in range(n_batches_test): + # Get minibatch + X_test_batch, L_test_batch = test_iter.next() + Y_test_batch = pred_fcn_fn(X_test_batch) + + # Test step + acc_test, jacc_test = val_fn(X_test_batch, L_test_batch) + jacc_num_test, jacc_denom_test = jacc_test + + acc_test_tot += acc_test + jacc_num_test_tot += jacc_num_test + jacc_denom_test_tot += jacc_denom_test + + # Save images + # save_img(X_test_batch, L_test_batch, Y_test_batch, + # savepath, n_classes, 'batch' + str(i), + # void_labels, colors) + + acc_test = acc_test_tot/n_batches_test + jacc_per_class = jacc_num_test_tot / jacc_denom_test_tot + jacc_per_class = jacc_per_class[0] + jacc_test = np.mean(jacc_per_class) + + out_str = "FINAL MODEL: acc test %f, jacc test %f" + out_str = out_str % (acc_test, + jacc_test) + print out_str + + print ">>> Per class jaccard:" + labs = test_iter.mask_labels + + for i in range(len(labs)-len(void_labels)): + class_str = ' ' + labs[i] + ' : %f' + class_str = class_str % (jacc_per_class[i]) + print class_str + + +def main(): + parser = argparse.ArgumentParser(description='Unet model training') + parser.add_argument('-dataset', + default='camvid', + help='Dataset.') + parser.add_argument('-test_from_0_255', + type=bool, + default=False, + help='Whether to train from images within 0-255 range') + + args = parser.parse_args() + + test(args.dataset,savepath=SAVEPATH, loadpath=LOADPATH, + test_from_0_255=args.test_from_0_255) + +if __name__ == "__main__": + main() diff --git a/code/fcn_2D_segm/train_fcn8.py b/code/fcn_2D_segm/train_fcn8.py new file mode 100644 index 00000000..f8e4ee5e --- /dev/null +++ b/code/fcn_2D_segm/train_fcn8.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python2 + +import os +import argparse +import time +from getpass import getuser +from distutils.dir_util import copy_tree + +import numpy as np +import theano +import theano.tensor as T +from theano import config +import lasagne +from lasagne.regularization import regularize_network_params + +from data_loader import load_data +from fcn8 import buildFCN8 +from metrics import jaccard, accuracy, crossentropy + +_FLOATX = config.floatX +if getuser() == 'romerosa': + SAVEPATH = '/Tmp/romerosa/itinf/models/' + LOADPATH = '/data/lisatmp4/romerosa/itinf/models/' + WEIGHTS_PATH = '/data/lisatmp4/romerosa/itinf/models/fcn8_model.npz' +elif getuser() == 'jegousim': + SAVEPATH = '/data/lisatmp4/jegousim/iterative_inference/' + LOADPATH = '/data/lisatmp4/jegousim/iterative_inference/' + WEIGHTS_PATH = '/data/lisatmp4/romerosa/rnncnn/fcn8_model.npz' +elif getuser() == 'michal': + SAVEPATH = '/home/michal/Experiments/iter_inf/' + LOADPATH = SAVEPATH + WEIGHTS_PATH = '/home/michal/model_earlyjacc.npz' +elif getuser() == 'erraqaba': + SAVEPATH = '/Tmp/erraqaba/iterative_inference/models/' + LOADPATH = '/data/lisatmp4/erraqabi/iterative_inference/models/' + WEIGHTS_PATH = LOADPATH +else: + raise ValueError('Unknown user : {}'.format(getuser())) + + +def train(dataset, learn_step=0.005, + weight_decay=1e-4, num_epochs=500, + max_patience=100, data_augmentation={}, + savepath=None, loadpath=None, + early_stop_class=None, + batch_size=None, + resume=False, + train_from_0_255=False): + + # + # Prepare load/save directories + # + exp_name = 'fcn8_' + 'data_aug' if bool(data_augmentation) else '' + + if savepath is None: + raise ValueError('A saving directory must be specified') + + savepath = os.path.join(savepath, dataset, exp_name) + loadpath = os.path.join(loadpath, dataset, exp_name) + print savepath + print loadpath + + if not os.path.exists(savepath): + os.makedirs(savepath) + else: + print('\033[93m The following folder already exists {}. ' + 'It will be overwritten in a few seconds...\033[0m'.format( + savepath)) + + print('Saving directory : ' + savepath) + with open(os.path.join(savepath, "config.txt"), "w") as f: + for key, value in locals().items(): + f.write('{} = {}\n'.format(key, value)) + + # + # Define symbolic variables + # + input_var = T.tensor4('input_var') + target_var = T.ivector('target_var') + + # + # Build dataset iterator + # + if batch_size is not None: + bs = batch_size + else: + bs = [10, 1, 1] + + train_iter, val_iter, test_iter = \ + load_data(dataset, data_augmentation, + one_hot=False, batch_size=bs, return_0_255=train_from_0_255) + + n_batches_train = train_iter.nbatches + n_batches_val = val_iter.nbatches + n_batches_test = test_iter.nbatches if test_iter is not None else 0 + n_classes = train_iter.non_void_nclasses + void_labels = train_iter.void_labels + nb_in_channels = train_iter.data_shape[0] + + print "Batch. train: %d, val %d, test %d" % (n_batches_train, n_batches_val, + n_batches_test) + print "Nb of classes: %d" % (n_classes) + print "Nb. of input channels: %d" % (nb_in_channels) + + # + # Build network + # + convmodel = buildFCN8(nb_in_channels, input_var, n_classes=n_classes, + void_labels=void_labels, trainable=True, + load_weights=resume, pascal=True, layer=['probs']) + + # + # Define and compile theano functions + # + print "Defining and compiling training functions" + prediction = lasagne.layers.get_output(convmodel)[0] + loss = crossentropy(prediction, target_var, void_labels) + + if weight_decay > 0: + weightsl2 = regularize_network_params( + convmodel, lasagne.regularization.l2) + loss += weight_decay * weightsl2 + + params = lasagne.layers.get_all_params(convmodel, trainable=True) + updates = lasagne.updates.adam(loss, params, learning_rate=learn_step) + + train_fn = theano.function([input_var, target_var], loss, updates=updates) + + print "Defining and compiling test functions" + test_prediction = lasagne.layers.get_output(convmodel, + deterministic=True)[0] + test_loss = crossentropy(test_prediction, target_var, void_labels) + test_acc = accuracy(test_prediction, target_var, void_labels) + test_jacc = jaccard(test_prediction, target_var, n_classes) + + val_fn = theano.function([input_var, target_var], [test_loss, test_acc, + test_jacc]) + + # + # Train + # + err_train = [] + err_valid = [] + acc_valid = [] + jacc_valid = [] + patience = 0 + + # Training main loop + print "Start training" + for epoch in range(num_epochs): + # Single epoch training and validation + start_time = time.time() + cost_train_tot = 0 + + # Train + for i in range(n_batches_train): + # Get minibatch + X_train_batch, L_train_batch = train_iter.next() + L_train_batch = np.reshape(L_train_batch, + np.prod(L_train_batch.shape)) + + # Training step + cost_train = train_fn(X_train_batch, L_train_batch) + out_str = "cost %f" % (cost_train) + cost_train_tot += cost_train + + err_train += [cost_train_tot/n_batches_train] + + # Validation + cost_val_tot = 0 + acc_val_tot = 0 + jacc_val_tot = np.zeros((2, n_classes)) + for i in range(n_batches_val): + # Get minibatch + X_val_batch, L_val_batch = val_iter.next() + L_val_batch = np.reshape(L_val_batch, + np.prod(L_val_batch.shape)) + + # Validation step + cost_val, acc_val, jacc_val = val_fn(X_val_batch, L_val_batch) + + acc_val_tot += acc_val + cost_val_tot += cost_val + jacc_val_tot += jacc_val + + err_valid += [cost_val_tot/n_batches_val] + acc_valid += [acc_val_tot/n_batches_val] + jacc_perclass_valid = jacc_val_tot[0, :] / jacc_val_tot[1, :] + if early_stop_class == None: + jacc_valid += [np.mean(jacc_perclass_valid)] + else: + jacc_valid += [jacc_perclass_valid[early_stop_class]] + + + out_str = "EPOCH %i: Avg epoch training cost train %f, cost val %f" +\ + ", acc val %f, jacc val %f took %f s" + out_str = out_str % (epoch, err_train[epoch], + err_valid[epoch], + acc_valid[epoch], + jacc_valid[epoch], + time.time()-start_time) + print out_str + + with open(os.path.join(savepath, "fcn8_output.log"), "a") as f: + f.write(out_str + "\n") + + # Early stopping and saving stuff + if epoch == 0: + best_jacc_val = jacc_valid[epoch] + elif epoch > 1 and jacc_valid[epoch] > best_jacc_val: + best_jacc_val = jacc_valid[epoch] + patience = 0 + np.savez(os.path.join(savepath, 'new_fcn8_model_best.npz'), + *lasagne.layers.get_all_param_values(convmodel)) + np.savez(os.path.join(savepath + "fcn8_errors_best.npz"), + err_valid, err_train, acc_valid, + jacc_valid) + else: + patience += 1 + np.savez(os.path.join(savepath, 'new_fcn8_model_last.npz'), + *lasagne.layers.get_all_param_values(convmodel)) + np.savez(os.path.join(savepath + "fcn8_errors_last.npz"), + err_valid, err_train, acc_valid, + jacc_valid) + # Finish training if patience has expired or max nber of epochs + # reached + if patience == max_patience or epoch == num_epochs-1: + if test_iter is not None: + # Load best model weights + with np.load(os.path.join(savepath, 'new_fcn8_model_best.npz')) as f: + param_values = [f['arr_%d' % i] + for i in range(len(f.files))] + nlayers = len(lasagne.layers.get_all_params(convmodel)) + lasagne.layers.set_all_param_values(convmodel, + param_values[:nlayers]) + # Test + cost_test_tot = 0 + acc_test_tot = 0 + jacc_num_test_tot = np.zeros((1, n_classes)) + jacc_denom_test_tot = np.zeros((1, n_classes)) + for i in range(n_batches_test): + # Get minibatch + X_test_batch, L_test_batch = test_iter.next() + L_test_batch = np.reshape(L_test_batch, + np.prod(L_test_batch.shape)) + + # Test step + cost_test, acc_test, jacc_test = \ + val_fn(X_test_batch, L_test_batch) + jacc_num_test, jacc_denom_test = jacc_test + + acc_test_tot += acc_test + cost_test_tot += cost_test + jacc_num_test_tot += jacc_num_test + jacc_denom_test_tot += jacc_denom_test + + err_test = cost_test_tot/n_batches_test + acc_test = acc_test_tot/n_batches_test + jacc_test = np.mean(jacc_num_test_tot / jacc_denom_test_tot) + + out_str = "FINAL MODEL: err test % f, acc test %f, jacc test %f" + out_str = out_str % (err_test, + acc_test, + jacc_test) + print out_str + if savepath != loadpath: + print('Copying model and other training files to {}'.format(loadpath)) + copy_tree(savepath, loadpath) + + # End + return + + +def main(): + parser = argparse.ArgumentParser(description='Unet model training') + parser.add_argument('-dataset', + default='camvid', + help='Dataset.') + parser.add_argument('-learning_rate', + default=0.0001, + help='Learning Rate') + parser.add_argument('-penal_cst', + default=0.0, + help='regularization constant') + parser.add_argument('--num_epochs', + '-ne', + type=int, + default=750, + help='Optional. Int to indicate the max' + 'number of epochs.') + parser.add_argument('-max_patience', + type=int, + default=100, + help='Max patience') + parser.add_argument('-batch_size', + type=int, + default=[10, 1, 1], + help='Batch size [train, val, test]') + parser.add_argument('-data_augmentation', + type=dict, + default={'crop_size': (224, 224), 'horizontal_flip': True, 'fill_mode':'constant'}, + help='use data augmentation') + parser.add_argument('-early_stop_class', + type=int, + default=None, + help='class to early stop on') + parser.add_argument('-train_from_0_255', + type=bool, + default=False, + help='Whether to train from images within 0-255 range') + args = parser.parse_args() + + train(args.dataset, float(args.learning_rate), + float(args.penal_cst), int(args.num_epochs), int(args.max_patience), + data_augmentation=args.data_augmentation, batch_size=args.batch_size, + early_stop_class=args.early_stop_class, savepath=SAVEPATH, + train_from_0_255=args.train_from_0_255, loadpath=LOADPATH) + +if __name__ == "__main__": + main() From de2d672a161b71c2a330e2f7418edc259e13bbe8 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 14:07:24 -0400 Subject: [PATCH 324/417] small changes --- code/fcn_2D_segm/fcn8.py | 81 ++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 52 deletions(-) diff --git a/code/fcn_2D_segm/fcn8.py b/code/fcn_2D_segm/fcn8.py index a5f6aabc..0af04507 100644 --- a/code/fcn_2D_segm/fcn8.py +++ b/code/fcn_2D_segm/fcn8.py @@ -12,7 +12,7 @@ import model_helpers - +# start-snippet-1 def buildFCN8(nb_in_channels, input_var, path_weights='/Tmp/romerosa/itinf/models/' + 'camvid/new_fcn8_model_best.npz', @@ -27,90 +27,66 @@ def buildFCN8(nb_in_channels, input_var, net = {} # Contracting path - net['input'] = InputLayer((None, nb_in_channels, None, None), - input_var) + net['input'] = InputLayer((None, nb_in_channels, None, None),input_var) # pool 1 - net['conv1_1'] = ConvLayer( - net['input'], 64, 3, pad=100, flip_filters=False) - net['conv1_2'] = ConvLayer( - net['conv1_1'], 64, 3, pad='same', flip_filters=False) + net['conv1_1'] = ConvLayer(net['input'], 64, 3, pad=100, flip_filters=False) + net['conv1_2'] = ConvLayer(net['conv1_1'], 64, 3, pad='same', flip_filters=False) net['pool1'] = PoolLayer(net['conv1_2'], 2) # pool 2 - net['conv2_1'] = ConvLayer( - net['pool1'], 128, 3, pad='same', flip_filters=False) - net['conv2_2'] = ConvLayer( - net['conv2_1'], 128, 3, pad='same', flip_filters=False) + net['conv2_1'] = ConvLayer(net['pool1'], 128, 3, pad='same', flip_filters=False) + net['conv2_2'] = ConvLayer(net['conv2_1'], 128, 3, pad='same', flip_filters=False) net['pool2'] = PoolLayer(net['conv2_2'], 2) # pool 3 - net['conv3_1'] = ConvLayer( - net['pool2'], 256, 3, pad='same', flip_filters=False) - net['conv3_2'] = ConvLayer( - net['conv3_1'], 256, 3, pad='same', flip_filters=False) - net['conv3_3'] = ConvLayer( - net['conv3_2'], 256, 3, pad='same', flip_filters=False) + net['conv3_1'] = ConvLayer(net['pool2'], 256, 3, pad='same', flip_filters=False) + net['conv3_2'] = ConvLayer(net['conv3_1'], 256, 3, pad='same', flip_filters=False) + net['conv3_3'] = ConvLayer(net['conv3_2'], 256, 3, pad='same', flip_filters=False) net['pool3'] = PoolLayer(net['conv3_3'], 2) # pool 4 - net['conv4_1'] = ConvLayer( - net['pool3'], 512, 3, pad='same', flip_filters=False) - net['conv4_2'] = ConvLayer( - net['conv4_1'], 512, 3, pad='same', flip_filters=False) - net['conv4_3'] = ConvLayer( - net['conv4_2'], 512, 3, pad='same', flip_filters=False) + net['conv4_1'] = ConvLayer(net['pool3'], 512, 3, pad='same', flip_filters=False) + net['conv4_2'] = ConvLayer(net['conv4_1'], 512, 3, pad='same', flip_filters=False) + net['conv4_3'] = ConvLayer(net['conv4_2'], 512, 3, pad='same', flip_filters=False) net['pool4'] = PoolLayer(net['conv4_3'], 2) # pool 5 - net['conv5_1'] = ConvLayer( - net['pool4'], 512, 3, pad='same', flip_filters=False) - net['conv5_2'] = ConvLayer( - net['conv5_1'], 512, 3, pad='same', flip_filters=False) - net['conv5_3'] = ConvLayer( - net['conv5_2'], 512, 3, pad='same', flip_filters=False) + net['conv5_1'] = ConvLayer(net['pool4'], 512, 3, pad='same', flip_filters=False) + net['conv5_2'] = ConvLayer(net['conv5_1'], 512, 3, pad='same', flip_filters=False) + net['conv5_3'] = ConvLayer(net['conv5_2'], 512, 3, pad='same', flip_filters=False) net['pool5'] = PoolLayer(net['conv5_3'], 2) # fc6 - net['fc6'] = ConvLayer( - net['pool5'], 4096, 7, pad='valid', flip_filters=False) + net['fc6'] = ConvLayer(net['pool5'], 4096, 7, pad='valid', flip_filters=False) net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout) # fc7 - net['fc7'] = ConvLayer( - net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False) + net['fc7'] = ConvLayer(net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False) net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout) - net['score_fr'] = ConvLayer( - net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False) + net['score_fr'] = ConvLayer(net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False) # Upsampling path # Unpool - net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2, - crop='valid', nonlinearity=linear) - net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1, - pad='same') - net['score_fused'] = ElemwiseSumLayer((net['score2'], - net['score_pool4']), - cropping=[None, None, 'center', - 'center']) + net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, + stride=2, crop='valid', nonlinearity=linear) + net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,pad='same') + net['score_fused'] = ElemwiseSumLayer((net['score2'],net['score_pool4']), + cropping=[None, None, 'center','center']) # Unpool net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4, stride=2, crop='valid', nonlinearity=linear) - net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1, - pad='valid') - net['score_final'] = ElemwiseSumLayer((net['score4'], - net['score_pool3']), - cropping=[None, None, 'center', - 'center']) + net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,pad='valid') + net['score_final'] = ElemwiseSumLayer((net['score4'],net['score_pool3']), + cropping=[None, None, 'center','center']) # Unpool net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16, - stride=8, crop='valid', nonlinearity=linear) + stride=8, crop='valid', nonlinearity=linear) upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1] - net['input_tmp'] = InputLayer((None, upsample_shape, None, - None), input_var) + net['input_tmp'] = InputLayer((None, upsample_shape, None, None), input_var) net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']), merge_function=lambda input, deconv: @@ -128,6 +104,7 @@ def buildFCN8(nb_in_channels, input_var, laySize[3])) net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'], nonlinearity=softmax) + # end-snippet-1 # Load weights if load_weights: From 5a2bf9948f590015b39c1c2fdec43bb4fd1dbf03 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 14:12:03 -0400 Subject: [PATCH 325/417] small changes --- doc/fcn_2D_segm.txt | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt index 325f3049..f0c9e8dc 100644 --- a/doc/fcn_2D_segm.txt +++ b/doc/fcn_2D_segm.txt @@ -45,13 +45,13 @@ Data Polyps -The polyps dataset can be found `[here] `__. -In each of the training, validation and test data, the input images are in the +The polyps dataset can be found `here `__. +In each of the training, validation and test directory, the input images are in the /images directory and the polyps mask (segmentation map) are in /masks2. The segmentation maps in the *masks2* directory indicate the presence or absence of polyps for each pixel. The other subdirectories (/masks3 and /masks4) are, respectively, for a segmentation task with 3 and 4 classes, but will not be -presented here. +presented here. Model @@ -135,15 +135,26 @@ number of pixels in the union between those two segmentation maps, also for that specified class. .. math:: - :label: jaccard + :label: jaccard_equation jacc(P(class), GT(class)) = \frac{|P(class)\cap GT(class)|}{|P(class)\cup GT(class)|} -where :math:`P` is the predicted segmentation map and :math: `GT` is the ground -truth segmentation map. Often, a class is well segmented if its respective jaccard +where `P` is the predicted segmentation map and `GT` is the ground +truth segmentation map. `P(class)` is then the binary mask indicating if each +pixel is predicted as *class* or not. +Often, a class is well segmented if its respective jaccard is at least 0.5. In the polyps dataset, the jaccard(polyps) must thus be at least 0.5. +.. figure:: images/jaccard.png + :align: center + :scale: 40% + + **Figure 5** : Jaccard visualisation + + +TODO: reference image from this `website `__ + Code - Citations - Contact ++++++++++++++++++++++++++ @@ -152,13 +163,16 @@ Code The FCN-8 implementation can be found in the following file: -* `fcn8.py `_ : Defines the model. -* `train_fcn8.py `_ : Training loop. +* `fcn8.py <../code/fcn_2D_segm/fcn8.py>`_ : Defines the model. +* `train_fcn8.py <../code/fcn_2D_segm/fcn8.py>`_ : Training loop. TODO : import model_helpers, dataset_loader, metrics +TODO : remove /Tmp/romerosa path and make them relative path - +.. literalinclude:: ../code/fcn_2D_segm/fcn8.py + :start-after: start-snippet-1 + :end-before: end-snippet-1 Papers ====== From 04a030dd5119322d8688388c9437cecd0694b882 Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 13:27:32 -0400 Subject: [PATCH 326/417] old build instructions --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 85de179c..81252fc0 100644 --- a/README.rst +++ b/README.rst @@ -37,4 +37,4 @@ Subdirectories: Build instructions ------------------ -To build the html version of the tutorials, install sphinx and run doc/Makefile +To build the html version of the tutorials, run python doc/scripts/docgen.py From f17ad4a676b2aed27e9b13bd88e96bee24803826 Mon Sep 17 00:00:00 2001 From: Adriana Romero Date: Mon, 1 May 2017 14:16:41 -0400 Subject: [PATCH 327/417] fixed fcn8 --- doc/fcn_2D_segm.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/fcn_2D_segm.txt b/doc/fcn_2D_segm.txt index f0c9e8dc..ee1f6c5c 100644 --- a/doc/fcn_2D_segm.txt +++ b/doc/fcn_2D_segm.txt @@ -84,9 +84,9 @@ transposed convolution layer with stride 32. 2. **FCN-16** : Sums the 2x upsampled prediction from *pool5* with *pool4* and then produces the segmentation map, by using a transposed convolution layer with stride 16. -3. **FCN-8** : Sums the 4x upsampled *pool5* with the 2x upsampled *pool4* and *pool3*, -and applies a transposed convolution layer with stride 8 on the resulting feature maps -to obtain the segmentation map. +3. **FCN-8** : Sums the 2x upsampled *pool5* with *pool4*, upsamples them and sums them +with *pool3*, and applies a transposed convolution layer with stride 8 on the resulting +feature maps to obtain the segmentation map. .. figure:: images/fcn_schema.png From c98762108d1d3747044567df0a739437469492eb Mon Sep 17 00:00:00 2001 From: StephanieLarocque Date: Mon, 1 May 2017 15:57:11 -0400 Subject: [PATCH 328/417] cortical --- doc/images/cortical_layers_net.png | Bin 0 -> 28696 bytes doc/images/cortical_valid1.png | Bin 0 -> 860288 bytes doc/images/cortical_valid2.png | Bin 0 -> 441901 bytes doc/images/cortical_valid3_v1.png | Bin 0 -> 367426 bytes doc/images/cortical_valid4.png | Bin 0 -> 405922 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/images/cortical_layers_net.png create mode 100644 doc/images/cortical_valid1.png create mode 100644 doc/images/cortical_valid2.png create mode 100644 doc/images/cortical_valid3_v1.png create mode 100644 doc/images/cortical_valid4.png diff --git a/doc/images/cortical_layers_net.png b/doc/images/cortical_layers_net.png new file mode 100644 index 0000000000000000000000000000000000000000..50c7ea20641ac0cc849bbf6c22ea4b5e9fc50cc7 GIT binary patch literal 28696 zcmd431yoe++czqRN_R;Qpma$}%FrSp(wzd*NH@aJHK-tsbhmU#kJ8yx z|KI05?^*Bnt##gYzQbBBXF1!=?AiBq-Pf;n=u1TzEOavTJ9qA2$;nE-0)HRexr2O; zdLO*P%)(f_bBE@RoTRv_oBno+r>2_Pd3Q?!8T04(bBjz-Cvo>L(vGZlFJDDrGev$B z{lF>p9Wz9fhM-1nBlz=U!3-qyR^h~F^KBTGg0cQ#SqX0POl<7>v)6|H|M)uZ|0|1jp~xb5?uYnM;?h87#PRbnB*6hDe*b?xV6YKR?Y4*{^!0^a z*#l&4pSXY$*!K45y(;IM@?b7;??EJ_!+mL}I-Dn;_GEX4d`q1J)}5D*o)N=>hs*HN z>6HHRu8G6`1@kXSD6zR~(j$T=7iP0nPZK1I{OIbw(|zp_cKA*qN2Pm?-&?2Sb?HdH{8pYE!iFp;F7k~f8}^VYe8Tto z-nTd8@Ja{%X3<0|!h^~d@`y$(xt}8q!CahZQ-UV=6{n=uWA9Mp_e$rZqh)q*0*P}6RrI5!CwnLM8eMVKOj+W=<%`~bQ!bCC{#eCygTlQuo!PhA zJ5~x}P*U&zZG?93M>#(J6Ev+rJPWMl(i*Per^z<`IqQh+^CRVfYhsoU8T%b3SK4}` z-!BbW3@&M#ouaHgVN7M8^Q(~ERtwOP615BcVqitclIZCnpa}4q&CKyD&~ceyw8-?P zQm5ApruEcX_uzz3VkGc_xOyhVA{aOJ*{gUyg3pxD>4@I%Vd>x2Jm(r*yT=MO)FKqq zL!ND+^mdc);AR+pA!T`cbFEopODQZYOejgGK})0fIWG_O`ue)oZJ*Ya77||B(D#0K z>c@73UV}?a2$4O#hC51pJcD9#$n>-}euXgXMrV8Yye9qqbxi3o8maHovt3`_8O^zE ztKuJTBF!r0u~Y)V=e`%=;g<*HXMzQx{WQXSB{jZx=ZRh7*@|Pm(}w?2doUdx-QBwn zu)j&olRao6XK$Q6vAaio~foMK_R_T&Og2r(LZ{$^!43c0jHnvK2mzq zq!3oJKWB{lG~>DaS}x@G@83Vi3Q0-~TKZ=?)HQOI(xNUez1-a0Awojrii(PtBKQ@J zUt-`t9ITeIAA~SFsW8IoaLLKZV^UM28XJYTXKJFmQE_l&3=L^!XJ^Z7X84i?9FXNd zu|I-f;8j~^1#`&~X^H#qRu`Tw0VcDysQveuVG z%4vA#`}gm%2=w<-Q1v@XZ^!OR64f&iLJJTzwsUQ9Y&vi_+^?+-T3cI7dPPY=F*rDA zc-dmo9o2PHtWZozNvW)-7l}#^u9Qj0zVkyi9Sr#)s}ti-g=d0Sfz`@4#(SCC=VVSc7C}yyZt`B|NcH%)kYmw zx{Qvur}lo^_+a+-CUE27<0HGex)$j*5Nl~^brIQE5!HSE?3X5A`NqSy-FfH7j~^jn zVbB*ZFoi`#0;kPOr9DUkRAZcZ&RNl7&}@gZN=g@?U3p|f)PIO9I@)5kOxpVgbZB35-8BKatfP&(w!4lXTBt`~7+ znPba7_fkcVZcSEQ@)-nl)|?-n%g$U?W@kSSlSDNWt5MpeF;(KJ-ZCH?qeM|Zjrys@ zRjccsAOweZ)@pq*CYl=hhPOeT^B)Ue;#wHvxASj$Q+L#)rmTEtxv__jj1Zqt5sx`* zPd=!|ydJq21LgXW=8WahGpPD=ICBL$icV0i!%9#mIkV~wjS#vlR2`C_U5a>=`@GC+ z2|ZcVfo%Rm6CMQ>Oy*oP13EB6?2pA4GlD}s@TvT85t~YPG}nf$+}gY1l#uw+8>#=3 zy5XXsE9-MDm<+Y+k1jFwE7a0-jxY9v8{uR-_@UxZ;&jt?=^iM%h(_4geBV}~a1*XB zD@@*sD>+%MYEb3{eTNyjs}^FI&R5Pb!t;eQ}@8RMs6{ z4;<5-wD)PoHoan<)T^e26yDM8fkvT*ef?Q(q8)M!J-b(=Uq^=4L*TYsI;Pt`lkTYO zRZBV$;d4J+Wg*Z}!`wtFA~d?Io_9<6w*=AqC+|6bWs@w4Y(6rzMLZLm@TqmW#)n=? z?Cq~p&qO>@<8my2ej|VPy|+LmZ`Fqp#k!5FAL|B-T$2QyKV;lP)Ov;o`}ejaWhFyq z;jW|6peO3HVVu?Frgo2Cvi%$?lqnR^J3XeYCCh=AzBueOlNhVhbigYQEtiH|^zF6l zl>4@?U*JG!1(0*uQzVy#Lhq-^>`y28w&fH>?_CAThn%*cPKLdfM{G+dbXuL)6i>!Y zGx0ZV%d_y%VAgqxE z)3VJ`xctIuTK%kY<3r@nv4&6Oo!gsf(^geKr;Ay9cQ&ru?YF(SrmL*XCoy-4gX*wX z)Fv7Q6R!u<;}Xh5uMP-aP;G@rhr7J>9nu8NrdZ8%2wX8+(_`_n>X*NF_!l5{DpVQk zGFlXr)?yXPcV$)(t_FFW9;0aDQ1y42ae)*>_>p*2bex?xY9>n2Wc`689jsrY+IB9h zNK$=bFv=Z2^+NjKvXUk>>M=Ii ziC33L*9Ry2^P5EWKUW9al8Y@EN^~Q{m9W}0x?7<0GBbEB*3HFm*Bx)LH|JHcnvOSR z_|s<5=X~CtdD8roh?Dz{wUMl#rc^SyAdU+F8781@MjWBcM^3Q5~Y~10vnbVLD#T7S?spRL*6bAgP93 zoe0!7=sug1OxJ;~%}2s%ORNe8#=27)MyO2K;66pbORCh7Ab~Gs{a{77G>#JQLm(OD50vz&y8ScgV>O^JZtjAUsIhf=te@k zgW@Q^?%$58I#-OJZJAB)8G#v-@iAz55pZNih~dtX&GY5=r7oxzJ|qe|pNH+_WfA9n zDHc6)h>R>JYZ>Uc!vRW?B9s^<+Z+sgsg(@lF8CFs40<{-aOV`!^sW!^K+cUB@Bw!( zL%*cLSfO(3eDtO%3@<;Q(VHU4dhkOaTOt9~GfL3vKG_pudegbwL9++42tK@x1#qc1 z6w{kXXj8-x%4TXp!VYS5>W2?QPfy)^9z1MlTNgDt^rk|6O{%e4QgKniR30G(fk~g*xUTj)kg6#9?PO^$pI=Sms0YR z-~kjl!~yABj(pminrcmP!nco*572}cg_D^pyuA8S=}ogyz8Nq90QSlsjNhoQQ2zxb zW+Y#d+4ts~bYqhqWRcG+rG^_x%qE%h2Qe9&eHXs`-M1E8CjY=QMNv;%MgeL&!uiGe zg^HC!NREHw^&y9>;*4K%WW)aNA>B-P+SQ!ls*Nl(ZC*qc;cp>_@c&$t5CiXoBx;Or zCTiDWAGcP0zbg&dT}TOyO-O<#HNjx4A#pHP)G)B9^9*wPhY9P_)A7D-kh?Gb+~bdc zn(aOqa3`5>p^p6ap{v?WFGYRQfDqK{)9f&SNscJA4zy!ViQCN2s&KcfradAK69Sur zMh3^GV$y_0o<2#T3ylmcizpgKYdq$GtF7wq4bC!1qJ1~W4}&^6 z@hhCWRAV~34GV9N4q5Xz#n5k}rjDMbMrTt@&uyfNEK4X|a^IyXK&W$mfOmG&plSW@ z;D-#OQYM$d{{sMGskCssH^cMf*JTWY6YjeP@vmNuL)Z}XJn~!kaEZrAMMd>5gsQ6A zLG9+czby410M+w(TZvwSYzMl)`$A#l7rg`wimOUnG=C6kMB#YQgP0-EP3|FyM2OE> zzbyS>*Y3$x`;&W@3al{nqThFv18rZSVi63slV*h#Eg_lC^Bi`sJ8cZ#YY)bek(R!H zADhCWlA*VyDnY?&SL?9+?464-Xhe{hKhtC&G(LRudn}9W^XF~v?mn>pIS5JOy6qEM zJb%u}%29LD(eoa6Yinz$P=iCa#V4gEw75huO_cKN>}>Dg02CM>sQd_wVAYtJR6+c= zGYNBR>Hk6w@c&ulfPQR$n8a^q{M8SMk%fg0!XitIriCQ@O$`TE?PtzQ)O^~7dS?us zYHM*Q6d5#=rR8PgBUgL-XA%+t`!@i$lik#~L~Z~48M3))>GX53_2#b= zflJW!SMM%%5|_z+gDSHzUIvB-0r#<QSIBOUi!YrQ62}J?W8S=tdI!A-s*>NK%ag&Torh*n39W6eZpKXDlG(4ZT zM<#9IsCD?uBT_!&__ias5FRd_{UyRdav-er$M>GucQDqjz6_#5M3iEb>ljcjK{@GboW7KFSpL9f$^Rv#s zOh11xrTh$(!?#fnDfXvo{y#7@7LP(l_ENVa7x!p{hfnZ`Jm#x$t;<4_eNNq)BT^o? z?k#4B+E|6Y-ulp)Tei51Vc+}(`R;=T6Xox;!soRt{MY51GHzJj>4BND2BQ|AWHIq8 zP#=-=X%oNQsC$K?c!e>qMG?-WZ*uAv5uS{b#w6-Cwo63h!9fL8{g~mhoW^SQ)S;JN zvQx@<)iD$PAtAEuG5|4^+Lvgij&MmCR~629Et>1KAst(knzp4=cv&SLz+i)Xx8E4qNX0RAxs=D^W} zx~t3qag?}!UU%G{efSA?nc6>E>}Jx8H9ssNT&Kvaj3vxWC!eJBhmjryTy1deoy7qX$Ktae&2&NkYC3%e>C9# z92)>Dfr{HJz5tBslPD*gs63EyryOl+g~rydA(VehwPoDdsl zt;;4qb-ust&8AQ(hj`0vvz+@HQh6^?A=D=*YLaq8Iji<5Qa9fu@0%?w1c2;a!$CmNvz-;a%0Ewi~=? z{#Ahmb}eVv?nwu@B4Xc{e;QuIM3M=R$uwU;1BWsdQNaqPsA65sgIh%Gk-lTPE+zBy z7`10_q%lKsA+%jDZQ45u*9}Q&a9W!PZMmGKR(7j*$6>Yd$4C-=VEapnv_Iu0QtBJV z3}FwC?e8&5H_QW`5{nwXs<@f{>a2K${E-jCTz31yWeP}al>AAyWXwmJ6s%7t;|5(v z2on_@=1ALMBE*oRB$`q&KE=!1KsT+)VBLl zv}6rcVb&~T+v~X}pIV|%)Jy#&7=nq{tfhn;ei&nVSmHJdUK1UNPDh=-KC}sJwA%du zS9?|rZptqrB0|U`u6wT+;`XIV{SuH2F(igtnU`k=-32^O8N~+lDwy%u=aOnltJGtr zl(gpQnv{>6@dTZJb?iu)?98PMkn~Sd=`T69$KZZ2Bz$XXPk;Q-=hstp<=N#!DU%+e z0~nq!C+oELg0ZBm%AQKM;yy>?VfOkNrumz5{FU#`=n}FH;653{oQkW^QJG*+z%vHk zNoW{nsX*5Q1PsIN)+!E)h~^cl6zH&Fd%Qv;R|w{3S-2HwgI9=(p(>}3%um2@ivJRw z?uDNVDxUP~nL(!k_G%T>nf!OywD$a{prKB0${;zbKv+;Ez}U(Kwt8b2;WIQOD;zr& z{7+t063-4BEG!?y`shHR<95F!bpTd)lx-q}UlH|zVbTe|f`(BfDF*&KS){xg%uoUw z(_rq~Izr|RPKt7>o(CSdxoBQ;pSL=i7II@Up)T!1BFviPfM6$!-J7V3J>Mz+07`&*n7NcBa zU|!vL3l}&vkcA71*$rYie3!=vtAhbzMT6#Ai$=dHgsF+KB!|#Ow58bGMLX+1Xkr)& z@b@pXofDLlmZlXIop+0(AR!5U-4oZkut1%VkZ_V^bCm8HbO#)4-;}Go^}oU)zuu5< zw2$3VOrLViG6I8w4hnQtRN6I)rj8Q-g+&tIFz`xJg4^dTV96f#PdKvnBwH~+7O`nR zYWlIPjMMb>>-6es;{N`AR8-Vw&!2}qo_QYmN*chZh_sxh?1I^5Z%-axUS-~Kr|OTq zQMdoR)}*C2UuJYSMfWO zL<1kghkuaH3GLM<`aH)jyms4J(D@A| zFIt^K`(Oo^LNNdv=m+NmEi_MU?!M*w)iojG7zy<`b`6NBfuMPNoN(TM4AC)v0VSpm zDAs;|i%l>p>T`%o0=(^u5B!u77p~^Uuh$;0nsGoN7#V;*_OCFLzSSz2SA&A2>2vK? z`-hh7NCSIw23sBH15b8kOb)Wj+4Mz$}H}bUNRQ7|B95}NXGoq`De6n&E)xV86{b07mwy;shm2h$7 zk^^p}iAe{!J(lsq8*uCcb9UM4#j%V1{k*I6!DT{^?^{EtNoQoKC)gQb*7vx<<#*Gt zx#YKgHw|M=q1)wE{V`BoTEV#5qo%>>u8qQI23Ph?9Z@9c zrHO7%Ef~Lk#gpn*WKbBcoj`5yZC6MrnP#11sfaEQ_^B#(zeRe|E)8^7ZaNAU5(MQaXzeva?#b{}9 zT{@2?YB}xBr-&WE{@qw0l1R=bQok0`#E?7N+a>Ddw4ub}TZKleD= z9K!$u@Vbq0Vc|%wl8ld!Xk%j|s4y%B28NY3h}vp(ZWL_R#jM8t4^s|?boSM*F6pe? zTsjU8oQva~6|T1Sc05v2RDOPb>zQgSAmMx+&X$)$Kw)g;8@HN_I()HV$I>TiHeZDy zf?A9U&>24`O3zYFG&D3m#>M%I%wGP#)ihG*gaidM>g&lU1)Vt0`FMHvC!L+0p*xit zbYMC!KnRj~uR&kP%eSYBc%Bk!3!WeEgolNN5&QF8y|Uf`YwNt;tdC5;HG5@ysuC3& zdoBFU$swS*AQyjQJ$C=-X zizA7BG8;L%4%(XT>FL2EA_|O;Cw24iK>sI7(!!@Dn9Y*=@KRSdqCZ9Gu%X?oH@;P) zSj)NZ+qc%w;sFush`4$dwT<6f6Lfrh#IdolhOr`CTzD$Eieq@`LL;i62kHJ704u35 za{m_Ex)0u}L4Sjmmn1|9zhuVu7;6r5hQN0ENb{V}lqLU*M9A}(OcA(9K-@t65FfKK z_F_)AK0T*ix;LgM#1D$RwxZB;(y?W1*>^Cs&hK5uV}lkHNt(kcvZ{#Efl49vmSU*V z*uL)Ii7O2x+@aE*zOFtAu+946tNeuN(YxWI2gk#T{#9wmhDRBrrT ziMx#>CP+nTR%qAT&BJi?e|t&=&W8u%5@qM5oHB%KDN=BwDLcfFp3O3@nwn*D-q5_Q zHg{aVO52h76`WMg?3t}u)?f_;I;2`aYU2^U8#3wsg1ZxVfT7v+mKQ&cVQ=1`ur8=Z zVm;&3o-6g?6((_RGsb#Je^(eO0d=Ia)p+J+Ff;vYdZSr+jyBIxdBiOIolnV?{l?`Z z*lFpA9g0U&Ox~j-y4#c0V zp#;&<{KUE@uP@eo4zp&d&IS<8&g?*JKYqr<>$o+lT%xh;XHy{Q##4NA#S|n@1Njka z{EFEZNvChL5Z3R^Mi|J43SBu&!l%Z>swOOls&q82r$?@WmnP4QzgSK5mpz!MlRl=I_oKREcAH-s9pw@DgXS*7hVhh{o{Uad+A!>l8#CV-O<;;1RL+nI`Fx7VcZcpN3Th73{qzi%|$8}#5<%2f9&L@#N-c@8&0m6|i z(?DbSQC-Q#WTvutl;zZ`BaCCM@=U(FFDEGoU4;07Ja+Y?Bg(2rkU~F=9_aPj5r6y} z&5-A^Vcn?hU&?yR!YEB@`S>|`aGOhcchh;rn5iU!aavK*&=7s!I;Y+{x=rkjgtI~H z^lR=p;5o?Zf3QyW59=~k@u{|7Gt!h((oc@q?xo0tPkG+!KHRq-2-fk63F)#erk#1t zdvOM|w%wJcF_y)_Z|Q+gb%qpm>qrAq$zY{XzJZO=)pR~nUOJH7X)qEKy|mIo6nRSB zlaiHtAwE^}+G~i#(Cp4+8{DZZOn~2sDvC<<`+(!8qUSLAleHZyper4##GT065qj&v z6jOv8Jcl$rUDpo-xvPUUPG{}uJLpLRDIQXUdIzO>MyV0joO%sSz6}&M>Y&C9cUh<* z#TzEP94%M5#Mt%Pa2IXYN^I_^ttf57eMJH86DdT`su&rHo(=~yALQ9u$?vctCTxIwU zVVjHP=K$1MEYMsdtIw66&8Z^L+u$sCFJFL`M+&+368d$zedc4wBZD2>!AthUY4PlZ zI5_-wlF(4x2e_2n#w3`LS60#ZQUD$Oe87y}RK#4VXoudEOR*m4s9S>G&`5wqjD~CQ z(6koL2;PXGX~Ew2+sGmWl|DZ^ltUoHk8A<#@w{(SlLoU+iuuJRi>Oyh@56?}ZD@yE zkkGVlo_-m-i(kP874#BgWDF4_6n*jpe}8}f68njZyOFLg1@IQ!_x1HPH8a~RGap?Y zM+GRx-gpYsg{fu~QWj0ScW*nt+obCRB( z-pD*LI=W}sTkwRSGA)hzm5R#Fmx&X*np^$DGoQIKjFl2c`uPw?jcHJ$u|R{d*??hiae z<5Ypy*lED^^n+_v|CQ9WolUtvDvU z`tRLQgkVv>Y*9?fW_l|jLHB-y|KRIiC}JMxKiQ)Mi_2OVLA}0xPNYSVvGKHGwTgD1 z4K{FK1(~#}`c1mdjp6BtXeuZ}O#S8wD?l8BFYZlEy503WrWAiUNvoQA zF<}$yx)p`&m3VXS31+wvz3Em&K5ck(t2)~)KIyynjkwW6Oq8lhj8Hons^l(0X2fmK zemF0#8dGfe7!n>0pcq-viyXx8(*u18wQK7O=5T1UuJgSQxl)TgCXeV%!7xNONG92? zud@BwQD7z+K%S}J8=0wBZ0e7tRzkbatLpP&&sx=+Po=yzD5j24RSd;dtnTe0)QFG) zoInB-$v^A-pX9`q65_V(X_#QFw`I?2T_Xgf3+8uOoHmc23BkTROo%0FH_#|fkUIG_ zcSfi5lLpx}0ElRrlu*K#{OYUMS7&rD|C2i^L~QTZa%cE#D2%oL7oYrN)8mh@|GPN~ zkv1+4jq8ScKGZ12!lBIs-2?*}SY_&`KI``^1f3Y$51G%40BB02`G-^g+t>n(MeTPH zjc?YX2R4SYF@kWYqaq?OF)0KF&6MJOt+TwcJm#6NOxbVtKS+OS@A27-QE}e4TxSzH9Kz+S6ff{!(b?c#Kdi#ot-~-7Z?44 zg6^-}Pzt&5T-)tXObrd;0de-{V0yq%O@4l8cS&I(fn{0~*&iG;KJ=5D#8_LK3^<$G za|efqYU~VDRaL*>W^H^jwgTR};R#-=@jY~QO3(_W?Cm)XUNofPfHgQZ@s~ZT?UfWI zzb%E?Kq{8O!dmy&-sGnQ5D3KH)wLVgDPm${U+WH4tH)rVqo3CCB;}m&>rUwF>qq6~ zF?Muxd>t84$@4z8{L3dMlR5AnH(~LyLLv+(HBP@~6Ba1>Ou8_eE&B?r6cw@WW09E( z%T$l*+Qz;Zed`@aR?Pdmj5*4esXwjC9fb2K&oYozjfg2E4S{mua)0mZzRHA^M z*iUtL(X=9xq*&BrC{rk6u8qh-HIZcA#>)7GK3yHCd}*Me1dR(Wo03n|{-m0sj349` z>Sv7{;3+0Yq{Y`{K>5$knonE!tn76NIlrxDV>7GG0k~H>@mPBi1(b&8u0DIj@hWm=Om4l0>mbbU3*9m zNuE-Rs(9@cEU+*i=?la17^QYP#I$e82an zd6~d(Jpor|KJ||HQ;Q;CVTbc>|y8dsG1;%DO%ugEgyIv34+96k(Du1eUwZ6%f4^ z$131yo}2F0eGC3nO&(vSM*{6Q{0oiH&%W~*So5i-gcFhAC6QVYRxDtOH4zIRvs5dUI4%*C%Yjn6Y#p)#F#?X8n3EUUiDn<*Kg^~(lzO0M7(7X$ zQz80Io)2|;ORn>q)Wb{W>JKJTe~Dps_mgCTo*mS;XKNI{#i{SySD3kRWBob-ZEJ)> ztaY`#`Lx!8;e;e|YAQicqXE^#5NTpI8mALT9Sx{3E9Z!^yJ8+a~lkksJ{xo-5968N6g ziDm3L9a3qIqmb}U;0G^JiJHK4T1;Y+<+tdA6|h|ShJ z73Q-1y_584l*Y^Xl4aHV3*~;#+6=n;#5@DQHQL~&%&{;tLZ2I>e{W`63D-yBzWbFp zC@Vm%qb~i>{%B9xC$+&;jaIjw-_F3Qn6@&lqEW^p^XC?=)8=gGOnBFHgHRera=A!j z+jM~oBMLsu_&Xqb3ME(j%5{%?%GiE%SCf%2s~#oH!5Y>z-?WA;P7Gp z@X@2LSZ|^r_%gUpov0hyRz$e3Py>O6fsXXVwOGTHs9qMr0|Ev`P=8QFK+p&S9hPXx z$|dI_=9Z>3CHQ<1^E%UNgHg@&)@=HjP&)qBY#Q;sI(;Km&pH>kTIDXK1o##6+!hU2^rkZrPIltZac^`R7eAOaLCiH4_+JC*#Z+KCz#Cq= z5EmD}Oy2`MWk_`Nx+}cA2V-Sc=J zG04csUh3)f-MHA>e+8EE{ShUx)^FdwVSD=AxP$_iQymT#)(^G1oSYyH6ZP1H_^ZoH zcw(YuZd0>wgA}xSWL91+LCti-?Pq#0a5=pJ9JfoN=6Lb{lMX?7a(cZ(hBg)D$f5QF zc&anOUSCj7xt<8K?*0~Aa1ew=&S^8vQ_$s-Yz5>87u52Vq1?)nTW<=!8MCQ*E_+__ zMYO*K81Or8MSS4rZYlE6jlrFv($J&GaAKFq7ly3pk858glZU`M}pIoJbM^ z4D)GU@AkI?lB|1TTkh8Idl~z8mO}jm$|g2OvzK81;R4usybssPK z+0+7-^n4Kg(Z(F0!u~j(z9&9J10%ls>*A99z%4^+-9lkQ!h`F%1kHpxw7Mx^z5B~R z1s1j6x$`EgLJT<>Sxj+;7#@+;8NkIVVqf+ocEk6LY zghJzG=Y_VrTy#IDEbb!u;FYHGOn^&OvQ+oev@)R*H_Ug{)j z3kD!@65;3n%e|CWqFd*ml|@e>V1FkVo3i3zz-^z9y8DNWKX~h@)X0if-rNjkh~I61 zP+s!vT&SQ*-lnQ2Dj?vP5+|`E-z)lqnmkf0?c@>dMRUHa0d8Jw5Hs z-WP5WdwcdM_wTp$^l*_Q0BDp3cmu^sOPT`0TKPFZ9`%jo$KZ(+Ddn{jHM;> z`ERS{^~td@qU`K!Lmd7V_ru5cF-iR@DxLr#(KH55^!3AE^g4PVocW+wr>5+64>3JG z{pYeWTy_vd@CD{g9v&Xjo88Is%za-l`$t7Z<&~63gKLdwR!~q7cG1&Id3mut_7`@E zl1{6`9T^@jGasN54UCHdpF0Em1kxz95b6lYn0;z4(W(s6u-_2yJZAL|2sksIzVf;q z!khnwFLBJ9!0ONGd2D@p&L?)?Y^^r|6%FlOLxa$GiC#D$MpnYAtHN|&zWh3nJ;l@1 zvQ&S3aPU@AN(%D$@%_}))c>S}#K90K{BMTo2}wvY9oPFWf0aGtHrrFPQrB~Q`g0lH#7C%RpI|+o0ngL=Sa^& z`xy>^^2mY6eto=lrPtA!0yV3Gj=vFI5Yr2sIszmvd{YC@>pgC>M1A_hqxD&v5$`!yvSLoppH@PWv(;;%wHo%;<)ob4qn2WVXBE2p07b-yvch`CrYEBda_M7+%otj=n8N?t=` zP^1ixCb`%oJ<~6d)k}3FVL#9sPbMr&mUaf~7)5m>%%#NwI`eni>Iy5%X}daPOy&}Q z=bfjc%RQ7MJsA$)qxy~igx~aYnw6NYpf4Ka8ghDi7;V$_z97C%Gy5hR>6>WsRO0lv zH@UOG#(v1LN4g!le2^*JQSkbWh54y7Ej_KIEkbBh1}Gqnh!!>eFiJc@TJpA>2LQzQchGTnCi z#EewNWlpc#9!XvrpjdZSJTg5_&spZZno+lw2k1rrIGczJH)vLbdLOq^KP@t`keBhM z1XHQDVTpP}{>=HV_EWG^nL>M@RKs?RQ5qEO0Xx*zRk6arltG0ZUTCrIPU4F=?82 znaMUT2djr2MA{>&BSFRd>-@wpPZErkf8?=`Ht`dk&r?U=q>nb|dM1rsek<)%X*=zV z$F>|h-53eDG?N04Be!*sD@MC+@P2`nKBdBqIZMwA%`~4gw8zoEJG-TX>RK#{K%8Nm zxxxZmah(1K8cm(N^W&slcnuTI&|xNP6m~WBRNlKxht&oSRZ@{n*lj~s2I(5~&@R+y zy!hptg7RD$to1ewV`4*@C70`I45b!C;AEY=xlhVz&%hO4U~baM*NVrrWgCu*YuB)) z*|P=BkBwytUONcbyM80dt!yBST7 z^;`G7gl7+zhz;m7%7GA5{i@H$$HVvR!p-hCx`5ML3!{GZF9aIwU7vIBFBPsBKz1{C zLJas8)DaEGq5#*7f}66XuPl!D3SZ^rvCTt+BlP%(cd9w*DT`?Pu~LFFgQE`)XCD>f zFAX=j1D&2I;p*cLkzsNv1Cn%=b;VhIp7Hs)Gq0tHoh2esehM@$G--&;eHr7lrS()3 zqE0qZ1D^4-11qCHGwPZFCcobZ0J_m0V9=CZ872+ssJW zoNc#eQzO-CEQOP8W1+yZ6taIakHtQKc_gav=LBObA;F*Di1ZP+?0Apwk;69)ArzjL z@En(y8#RD9y;@WJ@`K(~`ITTbzzPEH{t7@>@E@)O^WeY~ft3(oYd(||1MxsH?d1xP z`s0smB%$N3*OJ-*E2L*RlH*tO6EF^X;aA+DZ%P6ZMymMC4g+MyEXxxEzv7pQMdjVw zh~<&pGw8T4w#{#V1;XLj3pnWL%Q%725fPGJUcxN_5qkQh|H;iHxc!i~ju(6%LKNgc z-~TyP$)KyN3&M?lz?FPm6A*B>M87F4&F35q1TtMF4=-xuZzB`l78Z=9-+6g?8_6mQ zdONn)*Mmn#RfTqbU(K-LSEQ`z7kn?GmG73o5s@65R6JAwV4|bKHZU*{!oq@4RaIs0 z;v)U**_~IfUTxQQpV&`-DdJD^E1m%TXiF{I`VsWx9ueiaB$i|gNBnV(czN zcdg2q=roonvTSpp0|F@dmDK-mIva$^0WdKsQ@hP)OaAxyY#>mW07JrkmJdu7PEm~$ zFJr}Kn!jhjIW+WISKr{B9%!PWCBTZ>P`}(CDw`k_`bp8xXFdWv@ZRTn=Dy*vazk#* zG$gx{(K-ccx4`wFI$f7_No&Hq!3xFL9Q+X7_K zA>p?z0AprH@z&Bo!}KEQHdAFIZT7lzZ$iz=Y^cf8^%_AS81i+F#!z$~G>3bmkwkUA zzSpRPI5;@>hCse5gn*Hmc^Rj%;UC#-v;>hyMiV7^^v3rnzNt|mJzfKo^+B#P{Ex!b zrDoM0fB>E4#2*mHpKbZ_CJt(rnn-nz5?rH!VtQ}hS#) zD4=O|of*5Nr~?)pQZXN4AXyHT8XSsrmz0z+va#LstUc>X;^#8_dgo(oEC3Mn;jp4R zsjGkJbr1wsjj@}n4`kD+rh!0^Kw|+yDc8A>fB;B?wVwI{-7oS6-~bmVCrM{#UO}gy zhDSD*mdoY9t_k~R7`itwJ}xc@@FjapU!0D1W=M#Lh}clxoM%r?KYh}0L2ciGVfX3d8h zUy9XM{r!=B*!yxeuCOMmK7IN`$IOh*#l^LEbVT@1C8UyG@^P))hsWt4VV)x7y5<_( z7ta+?s$IF6_~AoTWciYWzjP}iJ%u&dQ> z{v&<;N2Cf#%)z?d?+!lyLzwu?&V0Mtmy~VS6*2*KeqMHDgHV430&B6KUy(w9YcB-% zF&EuPVM2YKlZ1kEWIUY~9#S9x%}H^)p%a~>i9a$2?9L6}OygscbGk9kkKgBvKT?%-nhq$YOuq z@y1Q*SlZqVmull81YN{V2bFJ9Qc=uH*r2OFK*-HrIjy>?qvH3sWfjf6lY0UN+V9>4 z%#|~~#-s9{jU~U){7u&w)I8XxoW)`JMV|itt;UZ<&W2HUq|1bydwp-0HL8=el*{W` zmsEf703mhb=1CZO6A2E4`MI*tz7la|vVa+W#xgC}L7g(*i7^+ z4!?=o@$+&+*QwjMn>{eA&@v)tc1OK|Ok?~YDf9SYNzbaZ*wmu#h-DYtFSWm9EX;lW zkX6x}q+c&3l8EcEDB0F{k<1v5n(TMkw%Sknu#>z`@H)udE8UBw;WK&EeVo?rG+%4( zIMvSHPUb5PEZO6w-tkdIMUYKC`IP4SuG>(F@Due9h&r3Rgiw1a0R#{7R$^XR45%S) zKn+<>^`O!@tobZcN(DIorJ3+W%Xnz>lRlCpEVXsxPN|N^U1WjV{Ove)o!3}tyJE1A zMbzYaC;ZS|dNMDnPAj`>w4@5rwi6O|$np)XAxj(^q!NdhsB~kY+E~XDAejn|;z2*o z)n8{J!STP(tw(i>#m`({yS2?>-N+hzFC`!Re(MV)02V5^kAy?xGAWhyjbWh(`dXHbWmwgeTvp{?;Ix+?c3Mo7q!@8G>X5DpMri0AZd#m+>1dg?ogeim zak>|Ah_X+AyOJ`g;;RBeqm+q`WzA6{O6xRqtY*P0xKY79B$fKLgHzcRLK=%Al5W8} zz3rzm0}5#9Q6OofD$6777V(wvMk7*0#8QAnAGh+35PHqwgudIk6njU%p5WnDhk?(p zXhz|=#s_h{t$_+@cll2kFxUHUx-wP+ZNgrY)tp=M3g34p)A*v3q}^j*5%i}}fp!r<@Y{VVKxPBS#V_TDKAglrfR zIoSRbrV#P^n6}QY6^r`mSKkbn@e$j(th+JujOX>JDT1!c#~J0vzxvlP%&l=s_v)5p zL*zKopFR=5zrYkbp(tdu-9O7H|AFq_79vJUywhyQMVrNRns1VON-}>oeCQQa$>SaB z@Y)~{k|hA{6Bo38T>Bo35DEd_B&c7n-WHKoAp0xSNHFgg!f8Me{reFwi76XCf6^9H zDBxiiQ$Mu@pA!M9TN#Zp5bxH)fSOrhV^METGQ$Mh83oTv`iq6M`JL%kpARR30gn1} zJlN{Fpy;1repSIWOri44(~!?+dOTGD&s*3)HiF+ZgAHg5?}y7N(X_h15m*b}K7=Q= zPZr_PpQu$0lldX%<0tUhOi$4E7RNTf<~kE`!ACCB5xH5?L#QY%5S?JZ4_C7ZO$K-fIc;qk72oW z{eQK0mO)jnZ`&6H>F!RE?rs*bKoBGZDFwkrcM1rCbc2A>A`;Twor^B%4nd_uK&kgy zZufro|J(b_JM+vuGtU=>7;bLXeVx~J#_>B=_MiSQ(31Tn7*T;=U7YqbOk%C_S?B+m zO%T&aW8AboHo9yHmq64S`o#aewoa*Z9o#hkRs4aT;Xj~;C;Ejp)~n((Xw&R?rdx-rhAz*b35WEgp)bD-M#Z|TcrXT#XJC}Jelbh)zw~Re+d<9uFDCgJQ+zd?p zG(iJsBY_>H_aGPVvr$|#M9=&(4Jn(p-sT&&AU)kvlh?C=`tGT1!Cj*{rN_-51NxcrMQ$zCM!HeymlZ zK^aG0f*r7y;H~D@`HY6`zo`ypbD_t{YH>L(8_mH5m$RpjNP~Rpzhkm!M6|u_y{`-h z36Rozq0r#w+!?to#~QKJ45rHM38>VCVUb#R4^hky8#hgk9AZv`=Jcpn<k4o1&E! z+f3(>!%|Zi((OC%i;5xxi|F;4%~21S|7x#87(RykY)^67&Q_peU|_Iva)Lo=#Mp9s zI}#2KPS1^m`;H}WViT*#&SnCl%dsqZv}^}$ZK4=sJ-vJX0Yh_lRaFTBdLcw3D<|7w zxg|e8-;zOjf~R|GIBqtl)Mlz^mBdJVI72F8WyPfBox2PjSc4U%f95$k;o|lSqJtSy zR4pw-G5*P7E-VlTWMywJy4q%%k%MEP?gOAT$%u&$T&#p2>^szzG&eO-YieqShK2d( z<>mdyB;+mwfBSF>lEP;)<<}y0b(?M0{@Fb!Z401Rek>bK7rj{crmE_mJwdpdGO4D~v_^_h7b z2TD*uXa(dr$k)W*N2qfXJJ1Ob>RjG1isUClpNDd&tP5Pe-a|iqTD(2jc2d0^=cLWw_zUOWc*DA@I~ww$~@(R zzSEgQkwDxsYj2a(Z|Gno@fi6}+*9xLt`G_6e0`w24AuGiVjfb~&f@%Qb^Ry5JMBXI z>xf@f#dnj6HApi%0$O?6evV^eQ}0A8vPB$xT`yTU?=7HP6*>tbjb?nQnQhDn%nBG2 zY-fOuCTHtDf#?q&PYWl`9?z)N+x*3gH&-gn!gP`<;l$F(r20^fXymJ(r4^9f@U zv^j7PhvMtj9NydEr;t!Tbm{9$v()mB>@aTKZ+=#?6?hgkx$e#gGo{wcODxGd-XH2N zS@FuP^(p5leACgx(Br6%qJM#D2*TM%=IrzFOiC-Nl-Cc->O8FFCn$QI#7Bu%Ei<EGEZGMo)tDCy#%4p`zY9ih@Xc=Hf4UyNXY{ZPMe=8 z%Fe0DBt1>3mXmcPAvVQ@@qaGbry=X#E>c98sPAkvLAAER_59Akmj*@hEXFhe^WMP&& zU?@|p1JvwGPp_qy44aX0)2R*KRg&y>B}VXl{Umq({ag=zVJWfpcBq!*sH9l~)HT)M zp)}~}JPb_eB2IW|*hAKg47>A+3tI)PFw{kUYxV1D_NAmpUx7lMtHV;ulz}2l=J|`5 z6Q!|f#jl_1iE7EBsy#fUpNg7!dj^f*Kp|=Yn((;xxEYJ{`mM)`P7xzyQCxa(BBjgr zGRw1<5d;hm+BIk-x?2y1-)3wHq~&~5jpVps$7bjT&ncW>99Txtf5gWGnsgFm^F81s z$L<;ju}lLweh>->qJQ(02!JvU0%t)0luFa%ngxhf_kCHmNL~a`MxJOVKOxmPqt0NP zcjuqZV<8>t=LbPW!1Ig{sT%=!6KLJKi&7-j^gzYvXc8A`X#c|}8l7KufTM1$C$ba6 z&`(H+-UaM}dWOja!l&oNRtYlYa@sWqknQ!4Hb5U)GI8&-Cm8j!yrcricDhX56u=}{ zzbym@79FEt3T6g!-cs3TVK#5J7=&dU?G|21j~yIO-)UK;|avhjw>xaod6&I zRYC%>#NOf6ybv&`-2x+v-T7K5IO_oIPS{sebn zd-Pk7KKvneR%E?osPA9>VC~I=WYFBAmEXJjXyayzlp_=Zkb7x=w6qXy#iW?h**6G~ z?{C9G&`y%W%=&{+JEx-Gza1^+wljkypW)hh(dWWv8|r<$)5EJcsFobk+A@2I@BYo? zzvu`HbfN$Dc|GnoFxJMJZkMjpebL6can&eL&hJ9xT>18IFaj|r|FQj9KZWs+AuJv> z&2=zPXkD#{?RQ+9W%IolpCK(RoWQ(@fPkh?9&le`NXcMQeZoPw3_0g)B@JlSHH>kR zt}OGgpllGAIC8NNhXUr|Fpi}Y;Yd5A6B`vJQf9GCfD*(7`BLwjnw|}i|d*Rvoy5@y^RyW&8!gMx)@Uf8OK8*s&EG7(LbPEIt|4-cXj!^gi zIBpt%`5G!S)iz}RXun^qb%kA8(fmc`J5A0V(DQ@cq?sOJ)tQR(z53G5ljjcE%gse5 zFfsM7{43ztt$ zew)Z+l98bUN`3h7a4?8`r`1??=Xq@?#9pFuUh=mZ1_4tzo0&0$zuw9iO+ zVIjwTRn>M{d*9>D2iWAix1K+L4vL08Mh1>o_v-%?L4e#gthF}*aDY==u%F{r^9#VBMn!|JMrkf30BuyR2YuTAN+`3qBaEpZmqJrSQlW zmHbq$3{m{xkNoK;9AEdRG8YmNoI4!V3-xZoMiw&Z0x~}ddzaS3@|}V&Km3EvmGKXQ za+9v~bcd0QfCCG3bNp+D&C+hUe)Lc4^*Jm*}nC5@!wQtIzWU7U1%86QAD!RR`@ zJ%87ZqT;dfW1M{)ns}PDL?JE<6$vh!eY)xEwlZi)_6nrM^4aqa?=r#?b(CNf1M3RK zZ2~wZybu-(6_pIqIJK9n-CLym;i`Y`d_oW$JK_yA39)Yd+(`_K(jDj(X9G*`=nnCy z&yVOKu%MOmFN|HS1U#SX=_W3pboE3&sB`|t4@nA??N1_x@4U}FNNGu`SbPZXfahWy z^9ZMBb1_wc$mNL_N#d8pI>#0iJjn$*hhi`7CI?wAD*qx`=FiwiZ5h$5O`gOaRrT4L z8ktV+(9?{eXQVKFhw=QaI2YSef{m*@XVdwisq>6>&x1mFxIt>r#tHLax)@#yl0cWu zSFM_q+Z74B!E;ttp4xB5M{E6_I-RuF#NCFP{*)M4;a@<_i_S!zFU>U-_P4eA_~82| z3Ugt(N+%1_hUJMi9O{`{x9}Pf?-8&F)KX^}lauVO-{urAfsE#F;HD+5g8rlLVjB2D9zRJj>Kuo5J!eH*E_25^xuG=Y#YRkS=Wvr}a)n z8J6=}%e`)XT@2m)6-(73?f2M!DLWDaSs#gJ68k8iEr?v)mNoZWP1q}i2LReQhT(o+ zd|7;jy++9ix^O+*@b`)6WS${AYfKgSPZB$}U%QOmtfB?FY-N^7S|#h(sO{<|=$XvC zTJ53O?FpVrs+mZ`uVna9(DmpsbuGEAW?OKLp@;(pg%|T_=8{A7H&QNDdJg*16gg>^ zBWxHJktSc*7&$vOmTI8>4)5yHLb99ypOwK7rGX~aht;ow_#<&yHcqerBX;P}A%CJh)4mB?=Bf_a1Uvre?-2t`*ED7&MgQv{Sb|Oh} zIglZ4nd0j$>!#!O`7(P4<3iEYv1mUL+}qT5Ruon-Y!^=`J`jEQs8%!Q3tpE`li2(& zJUsXC`VRY8#q=1}y?zd#qNbAb+j25-V;V$W%Kuz@u@|0rSXN%F-%|4_a*+G^i#kvG zQc=8Ji7K*zf$Hk5OGx-ZL{ZE5IxrbO7<68X6jAgHB!R ze@+5uHb8JpTdUk81V=Q5CMJpng}J=3dIB$@X-a?k%@k?@K4N>y+taJ zTcwOXAfn#_mhYfhc)r)TzXA2XACHX5I+_i|D>AIb&4$M4wiyg0Ka_%VW`jIQBhd46 zEQ64XqyQ)8=IyZOjinSGhGtY+S-bKNLIL<9j$~}Re(%e zxfRL_I+mRyBc+dG?1tNQ@c0|Md4UFkpc1)D2C+7DOlDJb%4DX{X1PkN7tzhDm4fGT zFj3LXgF0j>^cfhac#pTDs4Ts=(z!ft;c^neF;O*D>FKz*l@yK|Sw&xBYA-Gx{u+Ec zw!P#{q6KN8*lt%p&n}a#2w30v#5z}@&v~yl`lDttYJaJ2`?qeZy0T-JWTESnXDAhr z1{>6kwRo(9cQm0N4|pTp+`92Ph?HECxI$|4?p?B|KUBtf0o#DE)uuIN_w%@2Yf(HQ zEU_U=zfzhai}o4lf6E+8GOoixuYrfvJ5j=2{r;+^2phvjpvH{SwR&xb2;RROA zvo0caPZ<_W;SQj! zo2%;BypPu9cg|zr&KGsPWw%$CtvN#HU z@uAdfr~TE^a@aA<_P$zM)o2iU#vYCc6skOHjSJVRp|Z}qyLw$1e7aecC@TytX6d&^ z-MCnkay(dftJ8a&+*wwR_KTySz;r$bcD~TvjYEQVZ1VhhkYTMucHu$B^TmeQu@k9O zeV9r5;weRjr)dgr(J<-uK=tIzXv3#S&S`h~U11;C==)D9BFOp-$|wSYRl3Tm%Zd?v z=ZX&DlQ&I^x=J(q?B0XS_|>6L`rr?QTXtkeGnjzg)$T^$nQx@6KTF z1im*ZG9YAFQa9jSNZqD8yaT!N;WNAR9P23@c(!^}cRPdbLQwjALpx2>19u+^RS8Jr zPSZf+HV%#oa|k0c*_?SM?s4Voxwrg_9?F6c4rCS}1*e7~v*Fc!aB4sx@e~onh*KlJ z*!Qjhg8~CXU*s(2C?*3;b;=dhpikqi#O(lMlrd9QPqYHy;XlIF5v@uf@1aMju;EC(nk5Zq61Y zI-^1Fg0FxHE10ne53>giFf!O-iGf$;s5t^xTc;ihYD70 z@}2|iy^KuvxT?GysS(=z;RD3S&dx3hKqQxpoHPLZxQ8n>U8;)0pH~h#k_o@nE*1yBpZx2&Lm-IdgL zv159g3;;gV-bGKR(aodH9N_(Sb5-p?LB~erLTv*Mj1|fXXBdnHc#(mNP}>Z>QsW4G zJcX5P4*+&Rz930Jb04%IS3yF3ip>*03N@PdDFhkFB7q0Op!q;=%#}u(Qe8fT%{+N9 z7Wo0k!vi;3+}#nR*YyhLTs%TCmO3(3!bLVi^useo#{)rCamd9vul7G%D7>4Pm{{G~ znl=a#x)3cun>|n^lFL$UyS8LPHPH4?2qZ6)sljAx?0!@X(&Q=l!$zGf#L6LOWisU6 zh@$da8zv-Jmk;ySNn?GL5jLt|;YJi4a4Whey>GX0+PzKJhkk7 zy9d9TlJi+_V$g3MkA$tYtOr5y8_lJMNi!9C)ziyOoH$up4hWwd`jYaDT%#>0MatO^ ztJ^+qq%54avpbYN(4&3DUjg~PO|T(${buZH&wF*{Yhe3uNnPu*eD&u@Ts%_g?HINv z*E8pb7p#qs1K{T=P$G)5U{0g6>X@hW5TP1pJbX4rWNFB{{@C#xrRu=#- zsnZw}Jdvc8kxv-;Uc?c|oG}PCy3V9KZlOUnCpCxhOtv$8_^>K>E*J9K;ZnNn7)M#! z%bg{L9t(uD?>#r9ZTLDpzF8ZVW>|f-U#deAHWKj>>*7p_Skvz1bsWVH#M;_zasYpa zV2TNzJ8s-;;9nFeG;?Z2TC>kV=pi+=&7H-HCy#5@^T1Iqzm=V zci3LT{wx^=l`vT*JLIsB(TJ_l5~cp619J5;~hQ76&{vkE8hTn^5 zQHI1CemQt#&^i6Fi{_&mzt}fek{PK1MKR-Dt7qXGc-#Ji!Cm!1IbUIZ9>p#e;9u%` z?GA@8MLlW0V3f8-st}k^jGYy8{+dQHQ^$n&rnjD6jH__zXCuT^FNy+LKkMOFf{o>y z3#ud@jP??m%(Y*1(_Q*&IS$1E&AdoG!omZ#@2cG?e=!s4mb{L#tM#I<-0r(@x9dHO zcm(qwj2;*s^SikcaY_>B=GQ@)b)6F5A6SGdt18X?vcps~NQZ%4eeDA@mLxuUko7%? z)>ol}AfJwnHSb{rNn3<@;nOf#U1eo{vp5iViWQTk3L{J$%*@H<%1BS`JXbP3KfxKX?1SZ!ULa3D#ZEx54ud-4bmd*93ESDa5#9!zgsGb>Ix+glYsvN D@3s;l literal 0 HcmV?d00001 diff --git a/doc/images/cortical_valid1.png b/doc/images/cortical_valid1.png new file mode 100644 index 0000000000000000000000000000000000000000..9f76d7b24a75de2b8fc33338d82d1ab846d1512a GIT binary patch literal 860288 zcma&OdtB0I|3AL1m1~)w5FPuGjPR ze7?y4;dJoMPrv;1gAYE~dE)qyvmbo0wdI2kwy-|gW+}NF==quD?L*?(;D3HlBXOLz zT=!l%7nu@yDkLBle-#vc2_JJAlzuhIQtN{cpy>gY%d3}DqW7g=#U&C0(qa4mqeg(` z`rpOi{rmo-N=gE3f8?nj_8rD2U*6{f`WEE5AO7jSefyxvm*N7>9y#`3%`Kl``~Q`a zk`w?2lSm{G$s2@E#)7^4{r$n7e+U2l?;e&K9>k2ql<0JiMB;(}?Bu`uIdYj8n|viH z+amL}kE^5y6heDXOw9tX!J;;&xzu*~uD=;uqpuQdcc!M=LO?e4N*GurOe*s*sXd_Hw=eePM-%Gsa9 zs8@1_FB?mb9&1`#)a1(Vg-6NaH^#Ey6GaojLE0th+UVx8v~g(h^>&Au(f6Ax2^&Us z7B4Al(@+s)o-Db zT`a0tpD3!>T&fz7m>zaVEz(b2ymdM3;Xyb7eVhFDGncm2`Ub-;-nuR~sBN)NUw$2d zsxylYdP3%e{xy?F0bz+7s!L#yB&AN0Bzi46SGYN*0taoT3-9}9Lh3W7A!9mwL}uoG zB`w?)NkY@od;{GH6<>#}Pg58B!vgrfMwQ*~kp|o1Z83@RCUYk$m&;h!x-8b_Uq#E- z?UOb7%(|!FwUubYveQdc`g#Or;ZD4u(8|?4KP;rtQ}i6dVX&IWV{C4Ey+~k#xxEfC zuTeYB;6-&Jo)5#7SvakbxswcckxC{Jn!LDz%D>D(`0)lCnd`WZj}=G=Q90c^*1NtP z)y{BwX+F-5#poCPnzE}iMGyBZL!b8uPLIuG#4`Aq{gnl@ZG_I3=kJJ{+h-vXs8&vX z7bD1L0pQoID%<4~9+mext8uhhO|)zBlY_v(jqtL_PH-qKfTOX^j`d2y(g5Q_C!3?6 zO7HPj-)K(O|LUBk9#{Q%O0E;iI8zB1f8T5`j^mJQlY1_UqN??sh*&sbo{i0QX1<@; z6?zLseDX);+|<-pX!9*Dr`-5*Z>StQIYlYqi{=fN{N+NhUsR#-ZDRijqC^u9g5*OQ zwUi4L%McmPUM5lE$WqSBJ}1HaAPRYSSQaTsOO=HZa0EPWXv_Os&5^To;ymw01*V;5 zqhRGm6>9Hv6UWPJcDRBP+hSDbahy8A`H$P1|C$M^aFX`tVm9JuY2A$_=9d>5mjERp>rJ4Ov#udcyR5JEJ>~Y&+Y?= zX2waH>?;@QTKmdG{G>=u?{#T3=iM#5!&lDXWIq>*zfbB;7|70?VeeT&$!YyRC8xgE zgRqUdX-!L>5^Xw7X^elwy>S`C#3dbpRVS@p-=S7qdoUoCTAqd(bNsd#nA%fORy}#! zimfP1?bszR>aM^FyJFuj^@WZS^NNV0_@$tY>ClRG%R@1diD?}k+1Y03XN9iF%WuJE z!h63bx0&2wmxosJ<#FSvg*298*69)}BK2n`*ZXw)q^PPOepmOv?=t|dw7Ozr78}9< zvHgEup6tpY&dmBNsN&y&THd{}uN397V_dE5-diiQV{b3NEgo%CC;GNGt?tp_oMtKH z=@V4U)0sWSw&3NM7CTC$-h&{fR|oWbRq*b+%(MnO3YJXDh)G)9g)P(9zr^sC(z&*o ztkqXXnel*JPRlfv#mUFggRRlV%Yj}5S@tkfN?K6zvTWL{UOis<=(PES&o5kM?+AA1 zGo{D{P9pa+a^Jt5c_<%|#ocY^&#Yyqo0dL7>@+Wh2vcebYhgQ#;m;tx#H(G3+3g|p z>ZyOWH>r>N;ZN=qBi*xTi9A#OU{bXbc8)|d`djw`D$*Uick~v&|9&amE7SqQKHeZZ z3|+g&FN?W2reTK~Uxf1{Nz8jVPh;Lx*7{1;kg}JQS-)Xw%3OfhIoWxp?#WUO(q&<`+@bq9tZEo9GhL`Hqx97-LmI2tEUo=&RAQ<}&?QqCJFg z;|ZojqB`EP@eH#*DPdP7KhG0ccIHFxHR*;_cr%B);8;8~a6N9@!xeAa0-UEV0*dS9 z5VzZPFXkbiSi#trunEBE*@(92SKo#ho)s5`hBUZ``ey#jhVgl*w&kJrGJ z<+QC#yqVIaJt-xK!cZ+EBMNR#d=YW3T=Tb32e{k3)+26i%0CKy61*Lg*_bT246bqT zm#&hF=`X>)yIe^=oR!R^IQ(vD0p~vb@{D8SaLfy>q?!s5*%#Wn@CM}p03@bXHE51r zfsL>2-?YE&oj-t{pG%E>>fC6G>j-`;!aw=y8Zr1~Kff`3%%iE6PP9RK$Y?6ruC9U4 z+i=V2Z%ff=FPS@l0#)@OoB#^-R=}Yy|3qK6kmh%Kp zY&;q=bc2u0Yp-7^(5SdA261Wi%D=p10<=@|7#mnJSL)n&KS|__ZxG(wkQK1u`ji|G zE(&I4*6d5;JVyJPyh*T|97L5v3&a?Cq(=2lb35B3tI?Ax`>A@60~iGY%Jag)40Bd? zvRtdGt&B@2x$37j8BY&}G?#Lu&lWphG-UNJ!M-S1&B^ye`s7zn*%d`y)ztK>SijQr zn!9`ZZ#^Wde(N6cf2GKEY+^V!4ll>`t^7QHgV*}}Ja2EebB=Q%RQ8fF zzu_usUH{kkv?x53>@{oIK8-KK%Hs((3>%KrHVYoOshT}fKx-c^Clhvq6K4+DqI~N8 zD`1xxtYLjz_7foXO##P|p`N=ur>SLC;MkV4-rX6wVGyO>g`%G}tor&|V<3%_XKm9Q zFjlwlY_x+sub$eC;v40of7S{(yNf)aH0jdB zL=X{?HJs95Uxmk-d!O54LM0;Zz~R(d&Tc_LAxxQEDJX-c_;`f!%P+VqMZW~iQ^gTA z^Ux3)z1kPT|IRMBeuWz-e`^=S1G28ol^u0xF5k{vL|Hr$Czs5%u8CZXHLZDq9okgO z!>e$WbE`hCA05VT!^Mw|F6wt9Lz)&1_>fH^diDe_&m+iYc~POM9l+?L<>ec`9HmBV z6FN%8CHv-jCVidQTrH9jjdIat{wD$?4SluqK>uE@QfG(7Y|$|U1y!wVV5{_lPB3`y zbHUX4Uy8oSI#dW))?5ZiBW`0W+iy)_LN*$^ALe`VJ*@gc$Be(^UDe8r&=)H=jk8OIvl0 zz2M?=#J1M%Ez)*)+X=WidOLDzX?a#u=PL5ZYA3R-(l7{-3nBNpE1oE^#gt`6KV>_R zjb9)_%7!RqVtj%dpOUwwy$wgJ&La|5_0)5rgy|;XdBvdl^D=h2?EAM8FyQ%vI5Mnb z)%&!!(m(3ELPl-0FBn=U2oLr7VsG5q;|2QI+)*m&#gT5pph?ThyvPN%9Q8#s#=C6; zRTz3=UVT_&Qm#d7cjXl3DS#JycH0g!MZ%Yi+THXp>!BDLjeZj39gj{z?m$y#M+jx2 z;=LecEwx;26LxA@Lin~ywX*R=Wy#_YN5MG!YPDR^*~dK{KnPBsa3%C_*=u9y#V#m& z6DlRsF%jKGH%>5E)bxo9w~5r4r86EWSchfRSheWa!>?51d2PzpaaBF3{l`MaQapn6 zGkeOpG5}4$mjPQ`p*k72J!V!6FNNQF*;qGDTQ;8eIL1r+XyIATY7!bqMn%8vjoYGRtw6t$(#9Xv|5Nkk+xR6IM|q;?goQXunmwQ z%u7R*xraEU3>|a4cz8oa2gA+l1}S(LfuW^t)VBIP>D7K1zq0eld*%J#K!yArGk=fX z?p$WY@}pZF+Zpu@p9AwZ%l&c+NYC3yJ$B+U$led!pvVXB?et zJShbXeC!f3lLB=flKt}diGyL#JEVnrWL{g-HSfZsG<{@ovgNHs>59@FrMoNB*GHuN zrK4z?1Y&G5LaGE~Nb4ajW%7WEb$9+*qdl@=p~g4BG5@elXBC(QDXI1kr1Ho$LSaJg z5M*<1Zw@+FnIf zM97cB{3{yM2cwT0hW$J9Y!enx;+d?!(_Y*Ixomn5@U`DLZYFzFc(*nT!}jW1g0M|- zXPl>_=^mkklt@;j?=a!78BU9$r}x+`%qufgmpUcNzv+fCj)6DBt&tVIO&5_6saZ5+ zqsG}Yqu+3bhvgR|ChB#rSlT)kl`)X1dPlCcqYP`b^F8?P?X3iyI5=csAnw!F2H{tj zVTmkyCHIPVC$eTKE`n#OALppt@C|AZ))*cf>5#0|=5gAbrY&SCj|c^S;tnINmpQ1m9)KC~jfn1Vej9tY4^CP5P+SnlQi>~iTPbQ^%2iMs;hekZy1tPx%O&e1a(CnbR3pQshfzMD!lsyuZ#yf*Hepeo zdf%{gW8G9N5sq~LGqri$AnD}wy8ax1bWnFNxtOuj-fF1dwf)O6GFsIjAlYi^Rn6)f zZrehQaSs`!4I1W^J=fsYob?wiU?6E_+iG?J4r)+qup_}sd<9(>XK!# z(=lrbxgj;kOc2IKNIZ=-$5^A^s14CY&V>$jnwTN-w5-X{G2X8F1|H?8jISyR!_N3m zm+m1q>*k4ndG8Nh*~;PkM(1{ql-)X$+i;yY0|@0JUoQWYWnhsqOm7d=1SCB_w$lF7de9|w0X-Ci63oJOsl-z8wUSCI)^1M2==-8U;+0IyMenz`5nA4{ZM`FgyhS2+FHN9+E&BO%U~CH zv*q}t8WB;{s;mC`i0(m3`_n?{Q@%F86qqK}MYJ!g<_LqF6?cnIcCx|==2lDYLX_Q? zIUK_c%gz_dp4-?}lyE}H1T#k@C}2_KEf-%-{)VulOld9XOV92QzGU*tt6S_{Ybch` z=|GszjcD-cDDv=19V&t(#P80w>YnUA;8No`D{IS`vC72?PQS&!jC!xC$Nbh!zmSw$ zCm7$4d_T!A!-Hi-m^t>xB;#Tr_np`U$b%qd2x^JO9ZwLAPOzIg22o1!U>HN=w&~nX>hXC#NWyGx0pJosFj}*ji^+TpP&{%ckG6RFP@@lYr zrpj?C-Nel?-WXlc;>}3VE_W>k06psh<8TgE@AQbBpechNCoC zQqI(Y6d#M91x@pvnULWLkYZB8j)(2D=_?_wkj~fK)9~w|9xM`EKb=Nch6g7-Xtnh|fS4zR*`Puj@pctL1z8E<>*)^W^HuAk``QC4g z*o6Fw>UMIPeBzhD6QdB{Vex0cfk|OAagrr%x2er{3sSbDdu$ON*oE4UgWGGV>3wtd zIKp}y;FvOf>^LDhLYf0^b7<2PNR`*WQYCe`tf9U|4pZz0+JvQ7VB^mLOeSWNxjXG_ znM2vSEQHaj>_>~oOSe)?eZ?TKb$hzD%5kptMook%K9k(O)?&_hOIqFs#Q2*?^Rg>! z5UO;*ALTuAPf)Qq?z*dLs(<1C00#XJP+Gb8(aPw%%+^Plt>b>p-xXKJDjap4;&C!ua=T?v4(^ z3Axq$#c^??!%8i5{w(tjGKe`TJl%Oq>>9i*8Uxw1uY!N?rhi!-e+dIBgUHA7_{LjR z4r!@46l}?z-nX`xCvYyLF&x^GWu-haovwZ>vh(RUr|w&gE@0s?vnu!Nn0e8}3|A%& zXleb9BER)uTQ-hx15- z5-Nufjke3$lp34XHt~<4D0%$v>{BS%7ks{GCFFvUox3gzpO3#AC`qon>$xt<)nMAn z#yfSa2brp&$hL&3hhQ#K$)k{r5zm?ouzmRVKA)AfrK`@;Eg(!2Guqk_cFHqZG2SD^ zRG8n1nH8`QD&Tn=uEy>FJfAc(W#>G98{BUraIS3W)OC-vE%G`i6Gk1zD z#uu17ga|xg&OEnxfi{v|mvnc2*dMttu8m!}{Un2Ut~3(c6oXa>Rkc5p^H^j`v9NX= z#BHqv6qS1%atX2Aj`1Opu%Rjb)iyyy8PBj7Fc3_f_ZOW()h%mBP{_4O55`)3&sM&d zzM@XmJ9T<&7~|_xUp1s3)`5b!ADq`sZ_0a#dkO*iabh^J#VJ_Ica*)} ze448|9n!XXD2vFjf{!$lm+wv4+KsOE&cO!upvz4^m){|pOHVAQQ;a1r*!(<9vUnBf z>!tr|23AU=DTG8Ylf;KXMm60~qx$ zG(8QHsCWz^BbUFs%LaRJadwJ1yas*3FChzh8{70o=D%zYG@hSZ*^IZz22}cwUFyZy zpk$r>%}%-R-~0%%gRZAtC$6>KE(DZlt_>tqY%#SQ_F6B#&z*;;8t5KD?!48(k>His zDlj9xfid%T{De0LnYsD8T&K1UvuRXCOPFsz3A2?%G^p(`BanCN^ypAhv6X9$u#Lk> zwTjkhv0fW?2aShvA*tcS4AHK%#wPi#?FZTol3))_y?R>Fbs+Al@trw8w5Her6YJ4P zHcs8qTthSn0^qNtIYOoTqpJ03&(bFDFr$b!J`s7z0}YZYR5ekjC+6jh@;_ zpaaXHSvbq%7~zC2**7)7zSb>Garh~=J!%mLE2p$y)P9xtzCti3j(U@CN;8$OY0@cX zUDf@D&D?U>;|=7{=E&8ciA@VlrQ}~LuimzB7WMnJm1~zeU`Os0DlcP+(@|$_uhHkC z#~v=+6b)_pF(Km}VIIv}f0k^zBzULQ$N`nbxeO=2J>t&1*-THj?IV9(k zQ7hlukS6lZ9brm%p6uvFRyBD{{b?+gZq-PcJlXbA?H1goQ_aaD3&r<%U$Xe%B>nTA z*FTqfoDQfkiC)}k1-9%Zo98s$hjoqbJVkHDO9{M)lg(pqe?AAONglwIXuC1EHWuP! zvw2K=z+I`5H8kY!1h^oVW8Myp=rn~Gs-y8Ewc>b^dk&^O?#I!eInv$fC0fw7sqGrmBU?}N6WU4xxUViSdF{+Ep=<6S;RUr%)@LvI2jtowo=S+iR>+7CG~UUb1}?SH@>u;b zg9=s^bcV%lq5uWt)56|nk~>IQ{RhNn_f(lKH~#>vJ;h5h{>~FO#X|&HS2)y{xq^d` zhR205ax(?9>~84&u3qGXar9J3prYJ5ykq$n-6YBNthHZvfPk<%k8M_xxQL3RK<;|7 zkluaom+q;P0U*PzZYnfPM*MITt$GlF-4u&mG7~EKoxhMW><6Oa!uFzRiX!qu4tw6c zP!pmPodCy80kgc)yygC5k8%%fn^O6nE?^4cKU-+i<2xegdIdu(kVXb@8wSozupPWT=_vvuEuyt z;yd^H(Y&z%Av75-#P6b#kZmJ<)~qNEpXV#YpxL% zGE`6ZM1V+|jiZBLjs7ex&{^M}bvxMOYxq>WxH|$5*?xO>CD9-Jm-=Q99PZ|HOzBP9 zjK9&h*YUNj7r&kPBGz8!+8Z zT(V<*`6DDu!Q$`vleOLXNmcEd8JP*6ig|h%BNE*MxA9iKr&y+csBx-vXAMdv``RTD zOFC50abcZ+Dd;{aZ4UB_T+6;-TI`^wa@TN=MU>y1kFQmwZFsk8J^1%GK5Gps{leL= z3_hq)3Fw7mj08$jmqxKK#eq!94<4C@tEuiT(XJ42nurcT4(|Z)LW0CnrDkkm{sc|lxC4HN2R)x&2O0_JkKvY9QEje z$Sy6liSq~*v%O+)>2$py97Ebw=){G!4VJNvLPrM4>-&FWnzUD@4q~8)GloUy_t}Y< zM_Go_QR?E5g+l+5?pHt`yk_2~bD4X<5jdpdzu=qyzRo+BNd|mf*;^b^0K5hA;jKfS zE#~i-33+E90{2&cqP~{%1ejJ5Qx-k3!!8uRtz~miW9T03E!Yt>MXd^9MKvnnv~$fo2#wv7d3B)c;lmOYmyFMwK0R6R~X zK@SYOMjyY11%2!q#q=K7nohY}G3&@!dA8y)OG5{=?e=}Ocq#i()dg0?sCOhUft~3f zWtvvqY}zyQh$2{Yp&KsXRm`-kz9c^As=|-t&~ScQ)@ZD~Dn*_gCUaX@CX?Xt-q4nN z0r*VJ&=TJ)9QUvxd8-akJr(rPfXdJW$Io-Mfjy!qP^uu@5#kdw`gE(Bwwb?H=0!fWZv9o_wg zFsiv{3xZwV$I3;wIR&@Df|0f%nPqW%yH!+|M+UC&GdOhe`<5M3+B3bAf?djcWr5In zi0^`%yGTCD#EpL>z84s6Xj;#b&%a0sG55tq{_^3!6;uqE~MltwrFR zASaS{HOaTc!25V$;E&{G%wni!8gqRcH_V`m{xDxMG;}JUjpF{ZVD;T!Bp#68_4&Z_qrQ~Y93QO(_F`tJ7*E};h1I3 zl}y~nUFTt<=d}Q5NyyVuc7R!(fX(K%fLkH3x53TyqE?8bOgQQtQziJELfV4{LN(Xa z4+j`gD421$L)eGSOYd~$26cCDX*I-RIk|`CUoMUeklFCapJ~#B)_)W-w4u2Q;HwAC z_Mr%5^F&(EEX_&Sn!0vDF}wEaM_2tI;a=&&(&*q;%u+bh_!X&3EJ8`-z+k8eOboft9Ma&*!CHU-L8k?Ho{|4%QGK`a|J*rm5N{Mxsr0iHUye zQZS}FoS3!emyWSM?8;^A%)M_*S6eL5#5ly{txiK^eH%A-^_7*I1uw!ZNGzOYIRHIV zmbF4~5XCprD8tq%E20NkR)zj~FC!ci&!Wj2fdbaGM}gh3L34|o^U->QcVI}Jrx>Ed z89&Do)`cU0ho>`*E2xNIDDy-nnbvIAU{sR-9u$;Cwi#DjPVIgD~nty%@QZ2_QPvZ`{Jt^Z^LUAbLXBvvucf0xr2BmaRM8TSNyJ`b3=>YGSba& zAG@L^gtfBn7}eOqD-1>^FK1P)ihkO(W5&g^i)&TTK+VP62Zn<*(`$pke!%;4=Eyg; zt*bM4mANk_|88;pp}OrzDY$gcIXg^e@w=O$o2$Q*a;jP}svK4&Q>?T_K4wxv)e zww_vU9)G&aIdL=I-5&IkA#Ru26r>B%0E8FMp#Z0(PE3)!!!KjIaZsPm#e*$K_S-L$~!)eELYsPGxL_^PiYhH zL`Wk`lFKU&uYVj2HBoF@PLiG%1|F)UsO$itn@z%TxIfjPAb!+`jfKV!ltN|M*}P%R zRbNp(fJ1v3=1wsjN;IlC8b9%-@C_AS0(gA!GT$vq|^5a(vRU@ zG<+mOgWR{E|9i-!}ki63Z0Kylza5=TehkfPMa^TQF6h z7iH7H5$$4nWKyQq=k%3Bp|jd@65FLtB9{4=@=D@;kdd9+@Y{rY+Sx%2z5XK%G;5)n zxqG7w_L_oGd2wiIy18!vnU=-A;Wo4yt*v6pg^?n;N8QI59(yMPQZE%PZyR-A>b64G z#848WMQz%znA$6a*%f!j{uTa^sVQ(B-h5Cl#ZpQ%t$Cwp`2cBByT?Cvxu7+hxg)YH z#_*iwoV7WlOMf|O&VIs#%tot@4;^5z6TxNqsTH*`ZjC-1+Qe86a z=_fwt@2DA6*kOGbS;UTWf$zI^;8XeKW1JU&^^ zs9Nx*Mx8m_{4R!i6&w-5=@y`E;GNQlSM%kv);b+dcHOEXJj9@F{_)CbKb)jJhafOioYqD5O^}{L$E>RW2$&I6<9$1Q;o;(#DtE|? zyZxp6=3JCCaPNljAcE89_QalGB>Drv8hhtpADusfL*yjD=M-}aUj`AH-pWnZpO?iJ zaxTo_qVFH(jCGUx^Zf#?&nLo-%|-DO7_`@7XTNi4U^S5RXWKe^u@hi-3{{st0Sa9J zpkxD6XG@kf+u6t7(7s*!H-;`)Qt5xSNO~GdMuOJ!-Dl{{HHnoKcshQfr6yOm(d#lsdanY1w=1 zD185Eu2YIBCSXW_dS1xLT>B0K<&8F8Vly=FxZf#e%L$+9hw%Js<8DCi=(DAvUsa2U z!)w!3y+e4r6)u&Eq;O(gA%;GEN6D!)(&cbWYKM${f#HUO{gQwMweN1n{sEyFzer%X z-42+R9YjQ8Kia~V9&BAz*ZB#W-X4h9`RLxaRlSfja@|^KN0F6t!6cWUkN`Vz^YNoC z6IIv*K&Ywru2kvNp+g#fJ~g74wIAxz`eYjQGk;{tO^WSNoAJ@=hm0MY!skM87uK$= zscA@^!=7*I-EEI3S^d}40eu}CEOFHehybp?KPx+Lcah|npWDXQI38PZYltdl1ht~d zM^2`8%=(AanYBf*Gr91^{GFr3-{9tA($o#QZE&hVeZ^J~0rG8;H%(U?D}39-biQjv zD-!?~UwelRN=vwOa;h(^1%h3EbpFwh8M#!AO`ZA}2zmx9z}4bTq~5$t@?}u9;LOGLdPbAeGOZ|(Zet;DB)=bWp((l~( zqdRJJ28kxbUuKaSA?1|DwbFm!K6XK^hf2FK^e62RPZg=KhuRK=)T> zzn>NrEP;PM7qu~W#B3)$or>jDP@A3!GrW?=&IFAkL$J$tW-{mloS$!qqHL+iU_I7$ z=wYPSa&m89=Svbiv=*M#<5b!5bSuf{=E%BE1L!Mz^>kJC5#qi6CYT{lc8`;$f4;00 zCt}%M@h{LIvV(tMca)Rt6{u_KSmbch>=9-B^#kj1ySjh{{0jry`mW!Vc5V$ev9jCE zPkDT?(`oBz|1{A1)r+WWg#b<>T&4L}BY}R1 z;Z!4~IrwID3e_{GFvNPr30|Ou3Lj}6zYQoBYEV~c6$p>?meTZ59>sNH11jDE+Dqm9 zWt(X9^Byi2<%xDCgub))3T;sI{<`rI6fKOHSyVq>nLqmKN#09-*aWYPH+!)rdxT6gpe-JedkyX1E&5} zo%}qU?M)#Q=GGs{A_`rPNWYgxx^6wXT&4Zw#X{ojblz7hAwW6B6mQV)js&)Oeak>r#Im5vPTjJyu6!Nt1=g{)K1@HA6X zC&Q1XF3;c7j+UQd}*XZe^QQ`C+qzM{YJu+4hZ`3x-Vsq+FwJy#AKKdVo~|V+2vW zmYIS@g($I&om(4Cw&kl{x*&Xt0acOL-P)B|#v)$X5;M$~RZ{&&(7ELbSXY=+pq0-h zl8Y|EEg0&lsbS6+7QR(W*sJQ9f@u4dOX=-hhlg5NUV%=&A!|h%0R^XJ8aB(f z)U-Qaww*HkjC+CNa4Vi=?*UBK$*@RIEFSl+9{Nzc22ZRyhAj zws~b0%<1U!x87Tju1f0U#lwX|{K6mWJF}i<|sgah`zZ^X^);luQ`mIV_6> z+xmW2Th?njW_{li^jP$)uqL8O%fZvuEqdmekOlwk-IX?N79FsT!CGYnulL`|y+kYs ze%1ehfFV2&N-Jp+CVH&P?2c^+fqu7}sI?M()Q6VnvPIAyx_}E+;jlpo!e(E6+$(z~ zaACf;x80bbJnrBqS%bcewh_ICEo_HY$l6*S-r1GQEpzo{#usLhsIm(-Wa(a<_c+^j zxrm`u1s$a|$&fXR-Lm#N&v>~5R!&X?MUzLzrl=tIqsl zWKYiQ#v1}S2R(xH#u-##3{OyhRZNwA-Ja##=8!?Sbwx6H_&%~BoE}xE6P08Sk6`@u zT>xy0e_lQVL#3Y{W08!j=bURa%^x{~qi1`cv+=M9Vne_4ZBDpYaerqNs%{A(&1{%+ z7yYDBl1=wx_3H|g{?}3W{Py}j7KSMfAN5SAa!u9<&O4dshQRz4*S)*k3VE5)Y3P)h zXV}=MUrExIdGvT|d^gXb?jH$R_OxHa&rAHs67w=5jyHIvvYUVr1;LmtziYp zohp}vY`|7TaFk#-)6GBQRHaDx+G2Q?SKPR}kR0S?TGX(# z%iIb%RrAP?!@L%wx;Z_ZIT<4zz_oxQ;r@b>Yg7#g&M*5I(X7)~1^NkGR|QkAl;W4$ z=@0in1QsbARs2yL(SShGRM~|)D!&w@bSEiPbIKXpO2Z@_0d%zBOPW_44=Vej5^6fm z>k6A)-Wt4c9v-WzV+%!}R|I|5(Qh4&&85p#cs5Xvs}(<6J1upN0QzRAo>+M$v2Vb#h*#Nh znRMFK#b0N3XUP|HL`O}nLw4DL*8-bOojMr>(gM}9qVB}MOEl3IMk}_eu2mmvZAvdE zFpy53nuocfFsUW%w(_hF5t}jFYe{2?x5QHbj>*RC3X^Y53%!BVwT0#47FZfSHyLrK zbZ0SQj*q~yXo$ePD;sy$sUjzel!1FP)Nzn|!y)iWgP2P+#x7s!?dUSWC zauGjwpCn$vt~kC719`1`IJCz)d zy!sNhZYq5uig{;8-`ca5nb*Kdc7yDcEoOtxm|C(h6Q}EN3R>|kK{(VI@OO^?`nDDFX|qevL79q}xB^@v9d;g`ztnTuHR?{hH(5hnxv6BSx)aaD3^fUpQaN zHTA03t|;@Z%V3F(pb_Gz@eWF@6Z6f{$XiX)!RV)6=s?HK8y8Bym9Y~a?j%rpcc3;INJTGf5J%r=$zL}6I zacla7|7DMCW8T{2$K}8;w)AQ9{n{EM|@0p_5D(2->U-F zg)ub>302(RS`#zKs%|UYdyJ+M3<=(HVDm}({)!SNz_%gXnYT8ZxY*Dgin%RMo;~lB zU!v*_w+`9FMc)GSHZI3qpqc4)rQb;%opKWKIcbjijz4l=y3`O$ERqma&~tisrt;eG zh^A}qyv@p6@2V9zUEotF)7T+5sYt_>hlbqOc*7@1+v!$PRdmmtY;bEyJP1%q^cQrL zdw;Vn%YUqhGdb%#M0mUtPu2Wwa<}X39%9!MXOL2}gWl@R&lFYXZxG^R+8h2z>4(S={!Cql+jgRUJ?~*O*Uf~dNA+vt>kfZ=hnT4xq zyIH53R&V$d2v|Ir6u8zu?@ltZb{>QOjtmu0mF*Tw^%jl(^U(9PJ`!br}ZzwYLd&K(Q`)Y>>zX?|JR)hTJgq&-;Sn+ zsv*(%H>5_lG5uNF#YFH@{PQzBMQ5YBVc5{$BehVp!)B0$3FA}E%dp%02IGq&5S&Mb zQfPN>4S-y3N1@D6L0oqPTV$UN2sW!8Ay1e#D~U~p1f+QpPm^dL!yVcb)0?lJyhpqt z{`ioeu`_szQ?W3lb+Zbg8Q1l@3S)*v+Q#PDe$>>(@0f)3fOv2^R z`*~Q4&~jmt?N_F^`~dYS=^3!t@wxfHXPgTcWyehJE8o4{pJ*@$x&;_o2q%q}%9=GE1C5xM$Zv0v@8CEe1fqHM6TOUQXpdJmDTQFgKO=8JBAKMIR zUj6cy>OFrEA{9N-V{$)Bx(8)DhZ4!_K5y;R=$OJ9G`~7(tG`j_nxy%*b^hxt!xOyu z88aN>(3oB}K2XfQ;EQj5d$4Oi4d`ya%>qdD%eJAfF1+6v9cu1prd$4nkC@~qfn{@yV_}Lxemc;D&Lyv zD<%1;|5QBzuP(baVz*j3GUN8G^iPGzSw1j~{}7?et`azVzPinHUd4Dh@^P`8$aXozFxY`cMj-LJ6A|mRwQKLhF%>>+nj6aMsHd7+o zWM2br<7*c=PyWbwBhn#a14_LQ4gAr@JBRQsK|nTZH6Q?Vv@7S%01yu+JFZKnEvXq2 z8?NCGyG;vO?Bt8_fj(ri6+LxCP?mn=lzvtv`~raSsx3D>VOh3{Y35w5sch!jC|Q=^ zMX(nz*{g#y2A=*GAb zarg$(yG>R$XS{HVz@q=s-AsO3c;}1QP_}Pm%v2D+O5WbNyC^ZVr=f(JK&)}a5+Zo? z@u1kWoOL9oxfhVjQ*@LZQpWgMxsOmf&}Arz?DoT{Po}_G_Qq~D^!q9cHPr}N_Ig0J4*VE2Y1CfqXA(|R6q$MD)oP9xuT#8TgWhJW@ zC^v9vMv)z9Ii-$<`4CyX7tpVadRj;BId3Z(H009`dQ$v6k%x_4iV#1Np|`SNnv~OEzWMxgGQx^NSWbyFpUy0q^R^DWyVq*%d!KQ^Mm$=Rnu z=~p!_|LiUb+Pn7Yyh+Z24fk8}8z^uuh%Y1XJR18s+Z#&pFBivWjF|)}%5_~j#YXs3 za`-}jwpFlKN@l{rf`;8iS1Yo~Dpuig)9m7S{z}pi{gkjT#ii`Qb7<=Eiq|g5U z`@6Mrt(29eE)}=ja?SszQ&9xZtu5=D$(khH+Uq!+c zo=ESN=0<5}9tk7;X8SwRnGLUPk3h_S@RqY%<$&{bWMuYG1qcR}(ol^l|q zP#`I41#5NB(JMYkv#2iEseiz;{>Xdd(Mrg+R`8&;f=2y9xcR4fd9&KVG?W!Lfp*co z6>_~*mr=`yJb_w{2pQ`0jYSU8$hE@_eZP1jD8igEbK}m6Sap?UvH1xecIz~O%AQ<} z)dl>n=3nAQ`-d!F3=T~d^vvQ_*3Ly8it0kIIaH?^t5ku)rOg*#}Mez>-8ARML4B*R%tEat7A zo3=T|KdBR&uHwQRZ&~jli`D>>a5@VCA}1X}daf93_$Lz|yI0@Q^;)sz^K)|L&48Re znfGB)Em3~bj8}iRqb11N$ij$@EKT(+unHRp`rL#Q!-?I8^{t#O(Sl^_%2@E3H@oIV zqo5|=AN=3T@8Z={_TQhl`Eum8et~akjk{R-B!*mC<_U)X>KRHh$)*KB$utDFo#iE+ zsso3%_wlPA-W8^O2KHPLh1V@WSmdh;9DmYw1RS#YUT z?6PL)#fwmJkmUKy+XzF|m~bJWq;;G=@nf1vX|R$*_ZqYMLta?ikEH$-;0cbDBK1vfiQNs?=klE8;2HQ^R~A{C$cf?tP>(b z{IBvlOUoZ`_7|lI*hY3GNPR|g-sH_%z^q{Ln zBWwBLt@t9`nixm%jw{9Rl5EGU#Zt2cVM9{ypwkWM+^?<<2vw{npKKbkfB(lJ_L8paVFiourz*2nV6EtNF!!g*IuW`4=_;QW^py zx!3bBlH`$bb!eu*Ejbh=Q&{8EZ-f@%AWDP-5K=uU zHUun}meR{go4bU2!@;Iv81*G?;AS|3*ZtOFtUm%{{Bz(!@Ptoi#+iwaVutYgx5$76 z;s|+zhiIDPe*^Zv1j7eR%ifnW3kbL6TTKHhw}BDi>i^C_lxbp0Mh(aUwTj%BO+BWl zixEZSlsg^cUfg6XE8n|$+;h+hnVytDtKWD&{q3YSU{BDJx@0}BALd8k?8qN-n0vMC zHrddT(U?7ogBQnF6>l4juZPgWn{mdC<46)@T=hvfFQewnJs+|+eBh;+=46ZFiHRfX z-5Fi@NLBD!wAPf*26v2`iH16cW7?3HzKO;`cGA1iq}ScBd+h55|I&g|gj+X^v0Aq~ zq8@%iQ4J5E*D18_dKtgzq2+%M%Z2O0bOdVSK z+x%~w_GP-mm~~E8|MOMSJo!#+J=~!r$aroD`?k8WPBnA$0->E4KmcVVjyeUSyh#wF zm*hkKNzbtQBxYOPrP?SO%FQVtzCrXpFJt8WtFSA6Mc}0+7VR31iI`Z4#)@#7+@}M> zj)L5=pgGx$fqN#s5a0yHGv@S)glZ|`LUvm*vI>b)1 zk#^o@NjMmi@oN#RI>LLiVgS^t7wDDM@YN*TScO^J*4sG!*GV<&Sl|<;AF;u81uq%3 z8f`Z<(1R$%LsyYk730y%=;<5Tin&-FgS~Yf&V||*SrwxjP1br9c)jw^;Bzr_V)<$t?urgc$aO9;_Z{^ zq@B>b9ONcv2;S+H#+nRVKl{&a%e(f8LtA~5ggqPa|7@YH=-&{m-*2__UV0TRP(}pb zY~HBLousUc6(wE3cq^vnBr?J=#m#9U!o{rbZ#2diE-lz23rvab{!py5j$PIqz+C>{ z(xG3gK{(dsw+Xj9e}hG(`7K>j-w&3@Tp2lSD&GtA(W0b5X9UQBRhHAv;qOxWT#LFt zJ1tEt$qDd$p?q#`xQ4rL+5Zi)&2?(wD|}Wa=A|gAA`&t2W4qK3971ZJDcUq!g5eGZ z6+dzUGAMI<?A+WCO6V=doy+O6ub9}qjExNij7 z{o{moe?Q&4>I)_!fAJ1J(v8_r<>pLi%q4^ zpcPVbSZ}4$wVq3wW0NpeYDm)EL?x=TYRWYsF_mVGX67k4OuRK1rU-iM+^C* zACV}MpSwtOE^bvbASe3tShqgV&aFaNc^F-X`G~{@2z8qhAtZl`Q{lk{n6e4s$o$Li#w>NZ2nQcAbba*L9-u{e z$~<9Oo$K;W5&B$lF+YGQxI6g4+5dWaEYp%9)}162^7&w%eu2MEr7>}cAICA-5Zh%6P@Xc^*c|lmB#&396fc~DNt`$!{G`W zkmY5L(L{5i1|B6) z3E9MfA3^)Vx3u=o787Cb==^5ReUbi8O6!^z&~5T7E~X`1V42s;nb7Kfyx?0hSD$%D zppg%uU>|D0w-sP@-(ILr4s^3 zp>4|rC4UvM9JJ&+?EYx2WDyM%{vEtxRhT>|3g+%_U%#Gz!&b8&IUxxrc!?AB4cy-# zhCEKbYbm{xjti70>R9hl4uauD0$BH22_6ojIww~gU2=h!zz4Ju|xd`Sygd{!- zO4G-6KJdTSF@KvI$j4_%CbFYjP6kC7UyOtVJg&}kkqsUQfxVD~m*Kat2QWn1-qy~? zh=lDQf^ve=SiGiEgu`U?csOi1{m$`LyVaOBfCzWTs1;4NqhKdWkmRwIt9&qqkhgGY zdFvVU@^Sz2vh_KW;=1#S&NWMYkO$Y|^{zads5YH=}F2XoG z8Bo<$iY_b&Bqur?&>tnWSji=%YUbZ*bfiqTmcRDl}I9&f8bWM8Dgxa_fMQjEvHx z3Jv!*M>jpJ0?49Rs ztxs2*p<-*)rJGcWaz}wMEg3c#k1{hpsrGEWL6%1nH3amsaJY}{!}jk_XqTSWm;S@k zOVxXPhXFhQP65M97avxA(vAc6LH$vpSzzEpf_zTLbPpHTEJ}v^0B_fp`SZTkzEoD zBa0`@`6{RDfe)8ibZu?u+29i~oIQTpNK5YTgiYy7b@Lx!(l2ERZXXy58ETQnj&JSE zgE|h+QSX*?KRj!05Pa&^i*#ObOyc`^xjJl?VzM*0;(rL9I+lHH6pov*w!h8sCtHie|j)r zHS=fTs-+9uiquXgN0rsl=aBM*5w3Gztw5m`?AXmu<${plBIEf|va6GO+|tT|CjyMH$ICl zj3$Ab{I6~eqPQ9J$8|BkIqo6&JmvTOh?s{(z8a|lwL&$nB#3B`_pbIXFjuxtPk7SURhV?9)dZuS6 zY)NPxmZhpYNsL%c>a!?CnDf=nx@IVCFD6)v+SMtmZhcIwKR-E(&24p=R@?-#eO;oU z@d7SGN{Yak`fkPB4!4tfah|qfUQ1)%f{Y0Nk>Xod#6{B)S$c}$TYhW;)jcPRGhMkz zZ{zShpvx#C?Y=N-0^OHVtVrOnV1Vmq%Vkd_}BqX5LP;}eQO`hb*CTC71x*ZYN%(& zS0m-Ix31?pi&X{cG&D%C0=Dqexzt`>nsr44tt_7QhMdZ*7`jx;=4gt#(P^~0t~4Lu z{V(Fa-Rx5^GW^0K_5D{tPiG-ll&LzoJ8zuScxCzUsIeDR?1Wh+9-~B<==^Un9f38i z-1YMJq^}Ek%?-h)o5|pat1a~pA^M~{(_=>f;oI?aK8$yc0w5Y`D}E20lgD6Lo8^8>+Pxb_0rg< zI6l%cX}~tP{F2QXYUw@;vY$Fb5>aPP@m{AjjwU<6nfyk8zl|OYuWPe?%A>R@NniUr zRo~%QtYt7gAAgk~Ee70YmlI{WkBWD>Shd?KyVj0_Nn3vlF0kkYZ$p;RWN*nU+tQ1; zfR>nx6{=saCR?`l!}#+b)brj6-M43^SSK@QE`%@UuFJF&CsIb8tJ6$1pB)QAH+vwK z-S^bX?ji<++h6W1(^b~<&0OmYm!+^N`U~jI^bZ$5lJD*MfB9gi%6ff)S!i~mt?b>( zwpX#6|GQfn&&is=q-=gGuhM2UErzEvN7hEiwtE{0yEam`%?~DBwiCDexP+%!9g|fX zgC*3nZtyqRt^+5SW>!dOAlOtJd+0J;ZOGT4_Oy#BS>az;oa;Os=AoJC!NArLx}<)P z>%fTAVWWR{zwxMmhIK6sg3{~(^=|D25G`+QrSz%>8u!xHLN1ozhAPEgap_?o6jS9k z0^55=n>Z)FwmGAL3z3CwG#Zdf;-hJrir~=9($XUC#lRfz$mDH>o>kp3?qkHD*UqJp zyj6i3>wV5S;|=BILp3FBGjo|}Mv^jh$b-N$d^y@S$9WYjVKC@%lcF=PV%?|>p?ooXw}H=MAZPb6+OSH4&KQeuA#5DwolWr zW0&8PAh@4laiyM}Z5wkH-J0e!+-fw?_&VKO;b17~+%O(00`kvw(6ahmD1=N3`bjJ( zja!D&Q2WQD6Wv^L*0q3P=CJJ@1pe2nSte|+j5yZm-xm+>AiKLUk`{-wiWU1iqND3| zvC*D{c7bjvn2490av?&Le=wS9a{<56>*V=Rql&tBpxy?JH~xsjIcw*z>nm-6DDU(s zg@|nu4i82wVuu`cEgKcF}x z_?=ombMxHtJJH0Jd~<%8H}H0xRgG2~HAnKRoyfRVu8mBo3cd{rL$@s-;b@*{@|<<+ z^U%$tw21}sS^Z_a;6hGEMMR?t7(W%p#eK`BNN!M_%}Ge6&*^k4pkk<)$+MSJI^gv# z_BqCs0{=ZoS6uJQg@Gzpp-W{hwkL&Ags(L*BRV-2^k)=1Qc`v7nV%e zZQ}^Mikb&4Ul$1o(|Pu#ad~Q4L7ubQJqOhjYljHm^!k{Jwi;xp1LfsQQEN*>%y-CKrgePPU~4fPL>vuYBblU1 zto5h^V@2_9XUHk#OaykDdViaWgHIhxi`%ibwaipRKqh4WybF4EsO|vm@8#jGJ@7}y!#&^PN$66 zV6cOi@NkaLxNL@h95EjIU}r{VD>K}_Q3{ACBqv7UId*LdSqq{0Q{y4q$$3#~mf6L# z-U}i*bS$VIza%z(=q$H&!PW%RgDLIr^ar-v49z*8I^BE3O1`dn37{Z5tDkyqj=Ych zBP_^N5?)}_IB=k)t9)0LSy1Ph041+E*KV-X_nxE7s=}+AvzSn6>@TUh?4A$8qe!W> z1@vng(=FCzqXO-*W>XHz-&%IMw&PN?GB4F!2-g3}cZqFbP*6prVGZ{pnodq2S*|{z z5dMi>`5kld*YGIUv4wR5U2UMUV;Z6C6umyp>F`qvo(IB1q5g%i6`ZX#A*>#IGO+EooeXq32M(e{afu4(e*Q!?D0i zt@#{I{34#)12jW?s>Lv&wF@*7hL=gOzJwuirfE9Qt}LZ}-W|*vaZG_N_1q^tLB>%` zQ%YB>mQ=$bc1sK>Jd3_SdrrH5qT3kdXntnrn;K(lYlBD~9c>@AE)*;H(_iW-eZ~OiP?^ga4%JlYJriM6pYP<|7B(gM(l5_>KeXN;HEA1eWh)2+#-Gq@ z*S+mq8&PLgRYh#aB0FwEZ4frUYH|NE@=S9xnWrQFE8~pE)`QN#IkRp+ciNAF0^MsL zV_E2GojmQ)imr}zK?eA><^!q^nkahOG0gfgetweN<$z7JX>ISj_$=UW+i-_ailqN! z-?t|k%_LeCNeq_OJ@mpGE=m4sF;2IZRG}poYe@&~R+IR5*fE)O4)yeaB!E%JO)g7P zs#syG3Sp!h1_y71; zuMq8nj$#iWU?evyuCo|omc}K_RADMFxJ)ECMOG=+Ng`3&Y;VE9&_6Oh6h_pq#{SZ? z3$wqRS6hD;cI(_3bfhP|z3)K4mG=_4?;jTH;`G0bzl4OEji(ojTGs`62*A`6J4sqH z+|Gpstjk)CH7l#r?ZyEY!cc4!{!4~JRUea~j$Y{-Y4(Ievy^i{+<&$plH)a2${DXJ zsNeG+!p3Yc8*?$$w`g{(@C@vkTxeaH;edUI&4YD6zDtMIt~td<(@+WO*u%J4JP}G& z58$q|{COFDf$0RYyk`ykHMmhQ#>ZLbvMvXvPYRyS{uh84eZ*_AUokFqZ~u-8-?O@LT|Z~pZBsHMaZEv0^+^5eU_JwTqI7$Zy>|Lw5{gZL!d!R(6|c<6XcXJ}sh z@tV-%q!qVm;l~NIY1LhNWf~D5Sr_q9U%6w{k+zis0Y&UdZ?QeNVVubI5+=5yd zcokBS*QSaoW~&3?Wif+Aotg(x^7%ii6g54J1&U6&pXz@J73@D~eTN~cXT;WwG%rvR zE0VFfR8$ljtL~bb7qn}T4jw@i*Ye@ezW7MNqC6mMJ-Vsq+L(jrKAR)}o7seIa2Ku-FL^Z*5DM8Sp7Q4` zKUQ4wz@de$z^*L3f@{Jjl#%jU;ie7Q>!7~bbep)@q!OBCWuVqXdkz(TSD|XCp-z}) zZU)2)T|MN9D_#pGt;mHLsmstBZ!_N#ICD5So(+^Ve%kNIyTo}o`RH#Iq0zTEDWY>N z5yHodR0qN-Z5xN;^aFDgINLx4d>mwTXPIXDtxOQVC^;~~=;n>w$xYCYaRd)}SfScXi=R1~k!6LHut37`Axa8gJ@raiBj$QTC*tC%|4|@ui##5U}4S05apG)sBEUQ{$(@H zK_o1-P~~@y*0!noxyd%1V5BN{;PoqQqF&mWAjyRWTalJOj%!^%<(@_Kf^*GEr=5%)&X?SO zzc@AGe>N>w?D6Cn@hyILs&x;bV)}mcVuy6Za=o`tR9V=j> z3sza{)QGj^7Yc)Fu!K*%*RO>2gx^d4t-vFV@Iz2i1QKy~-WqjQPbX2JahX$!1Hz%G zc)&5*bC@p=j5iKV79T~eo$A=Eclo&`W0*ygyVZ1$FNjXneT0Mw_~2H@V~z+P)<{yG z^IV3|RsM}Fy18Fu1C%5P`%zm^Wfl8Da(wLdA{H1Oa+JLdousm<>)6LX`YOkcROo9b zjd2lQYpS<}yTh|w2t;y0AIk;Y?*6j~5oopTrOKyHViJoP-4yr_2~lGC|EwQu$oa>Y zjHEZBC+*Y8vqhtyrF(mcYDiKvTkyyIVC`bg&L$=$CofD$BgIugh#<+Qt#E}}SjgTSc=f*F<;zHz=GDT-U{@S7>=19MNPDRXXvU3pLC4J3BY#_UTu|&3% z<_|?>QRo_PZr2KadU&A?Wloy<#svjqs8#m~sd|NC8nf3GvsUCo$?C0Q2I(6KS<#83 zIixl&=t_Rwq8aFtAbH$u z2ffD(A?I+`NDh?_DFIV@ztZ^|aD{MJ$UA~cgN?fyNS1b~NWDv!j7}{8+Q+`L0JY*J zuA_%^LehVLHDEL#l*>sinm&XIme@Mwv}7#QDW(iij?fk1tHipVEYjcoLiDtv!96=8 z*hOYmoL;$Z6RC_&zq16;+7DS9k532gzXZ-4nM{t2>$5J+;0YRiS!sUF?P^Y{yWqDK z`f(;C;1m?kRgBq{k$!PbYted1h``HaB`po-%D|UH<%+O`=Egi3P@nX-GJiCpAi&jI zeT(vjQQPd)ze<-lw{Hx2S_EybzHqXsR3jIa>xrept;>{guL!%<*aK%2|FV(rkrj4Z zsE>dnUr=27oq{;cKb~gWehm&8$0~2%xP9L#-4}^yG*|~Z<>kt=ekqdsmV}39PIZZ2 z4r{&(QW@ZP3&6)P?Ta54QgshWpE`GI#wj(R;N$6m_V+FzxVB^CX3534E9ksvw zN9}w;xkr6AcBR42?a z^z6+Ij(<_coQlP05D*gV8U;B7D`4{GBBvy&;}!$&)X7&w>f}8gn=PZ*AyXZa9@|n= zRhIO4Vz@bAbDSh`W|Q?dR?Xc}KfZJQ|4NJgZ+TKM{=;tbkN?ev7JfktY^fc#|M$Mx zG?c?DMZDU4Aa_~aUeNN6Ki&Kjj0}nftjXJ3GZR%NxQyz&Ikz3(1`Z1gL^uD9A`bAw z-q(5Q&$9z5#f_7n1D@^mris4MU5w?iuge-#lZ3}JdFjjPHh}!6d-B(%=cBo>`{i$O zX{0vm@F-tZ?sddsd3(~pzEV+w09hOL8-7k9iE!&gugu5W@VHo)H1vdp*l>9QA+X&Q zUL>G$ZV{CDQnQ^wA4k6;wGvl1?4mcq#I%{7f4Ki}}2T zaDQ&P<)qq&3307X=(zki0EQlq6Sk5Y=MNPZc?`W&8eQx|M_V8~bCNYIo-V=c?;ia= zN3t-MaI|PN2s+2~yU!l&Pk$5_#|HC&QISh$8AX7L z3tO3OB(@DkicHh8SE=4@Z|JAJofekcQVl4{aRxYM4e;TiqSQ;W;qHOGqi%11+Fm$; zA9@-ia7x$PhZp?U4&1&`O3l)H0S-v+r5+dU(zDGDl46=htz+)BFP`*}f5YUo#WYI2 z+Db|%eh$xox$^=|EH2kBfQ8eWRA0?s)ja-7=?m{coo#p)cloTB>T9uh&zYHD*kM2b z>~oXN5M*^RL%b>L^Uup-rD&`{z_Nh@ZGJZIjpn=Eg4-`kqv}?gdls@@AnDOF-#9sD zt9wq?$UbB6f}-D_-ttKIp4OUUKfx_J*amNtV$vrcqfZ&4F%6qY98b7+;&ifQ;vB1#eg6H|=o){!7UN)^JniA8|{<3^!<{o{&Gj|?SzjZTDjMf^Is z`T5p?F0<}t;+EBTsxJtl3h}f$>Bbz_KT{qhdqyPl(i+7P zSr2?d0}GSlx1loQ?{J^H1LCy)zgf7gjQ0S%u?JO<(xji0y^rm`uz>#v(FS39m3MCl z_?PkPr^#$k%+!&2XI)fzSpCA`p$+1iX}804Iwmb%c-*g?)5UF(-&|g*`|@rrT)K*Q zxc)oz=-E?Kh_}w+e`Wt&(E7o*f3D=;nU1qdQ5!ig4dJcvc~XywUeH zcaCFcq(PAPC`{&hejLFsdPXXLFB3eDZ%Ozevx4lCaHq$qQTCm) zTRXd!GGSjrW(r56c&bTCy{8i}a345Jf;HG;*!mJ$4!iqe^NYh8G(&fMToDG>&!6g> zc9yY9MVYwEj0~)_hAGA|Bm=mI7IzR8KPqzbXhqx(t%=`?5MNY_RDOa=J-C2vm1?$I}q+9 zc9q=bJPL3z&(TX+71PS!J=d!Fm-g0@pJ?X5jaLSHaH&HkCT=l{x-W#==LnDV=vvR4 z;JO0q>ydN&nO}|6o+cY#n=~I(UIro!!o{PlfV%#HeRk#`MjWkdqRE0#tRO7WNPEEX zJWa~^yvkwKzzT^M3GYn{YUsDvK%VjEmAAz($AQ&+i(`z$F?)u=AZ+9|PS<63*ahN! zz9B&DxIORNA!IXljOo?TX8g$`!+aW-+Algeg!7E5;KW^-G*jjT%C{kKG!_kYh-fcC z?cX27r~`RSv}^tyuO_>61Mlq8BgmzTX6!nLdSQ5OAQBT>T_{)xaMS(pYF?Wd7iU9! zc>yN+3|vocb;Q7Egoh#beoVZCDn`>zgIvRO1=XwbmM1)s6@uPBih&MJmty`21bWWG z+z-H%E230`;+qhYQXl2jt^P%JM-cArB~GdqhFz(B_D+lKdlY{Vk=nFY0dMa(=B21% z%2T70qvlfYoDG)zJZW!@v&($^9o$$KNl#%xq)UdTVQeREnrmY-toa}Z7NnoL;#nW} z6Ftq2#dLwJrHhk#A`r$rR!qPv7>ewjq-^T5F5~u#jTp=-&c#UGlE67zQ(#-N)SkAQ zm}XX`Im&7X$|jlQX7Cfh$7VJA>cTxb24wueo`H!UncZidut>+N&JsL92{2ar<}Yz_ zFH=5;pAtVWgGHc+9u?dk&TVe$B7S>9I`2`*>wUmMPrvIn{S(!=T@@z=@W(!nKM2i- z>VCQNVR-W4U z)YyB-8IyKV9ZPiu6gbn)=B4$=<+k6F0K@KU~v)65=3BUr%ZJR%f-d7p-c_BTeD1&%bA)>MY`Bq<# z4bHDjYLwgxTWMF$JDSsY(}9jHa!>=U-^)^7xuGQqWd!YD^d*5`a( zt;_0?SkM21zR*_|*|nN6-wccD04w$ozLf!7xnn#e^XQ$&Z}TUdom)7(w9_hgpjgp5Q`v ztAKSxxzwPpB2*-|uN$a6Jx)dAYBy=b6iV;LM8Q(2FK>z4q0`cXIe<9^vE1ooOif&*-1FG6MlGC4>mSQqDU zqM?))$rVT^)T?-}P15f+zVS?ADX2emPLpEY^`WoX_B}UPTjiEm{&|K zf^g!&%dAQ-oJU$4@Y3G?a(`regT$_q``O*O{gWDmgZXw%>zDtkZ?oSRzLwv&4#nCekD$o^4wTVYSVE8dh&q9@pJ_f`cwM zDkH9r!$SP~9|hkl+Yrbuo&AWK2|SWkL(SJrnjv|zRRS4p9{CVS8TG>h$`gZN(swaa zjB((Jm-cot_U=UY`ByEiCfi8Pi}d)cq&B~3=0`0TEcKPN+$RG{H$n%utCNFdn!aUu zC8c&ha0!A%!Oj;kn)nTC6e;sM<@6fTK^uFeHA2er9G{2GwB98dCBDj5zddQmz7~zD zBLB&=*|57R)bSY2U$efS5|Xc_M_}yGVpT7qwZ)UD(xv`Ot?U}gS|=?46N@3Wh}QZ@ zmL8>g1?XeGQ46f7brLGpNR5EZM^hD6fX7h3=pPvXf#m=7MG#E+QsKW z*TU@8bMS4-giEY}iZZ;6AL9bVL1nA^AT%BGV1 zl~p$2Z7EVMxA{5f&o4#=A28e@W*ev+Bs&N-JiFk)hN!i3X%6+tINxvgd5abK&FO@R z??!VUOIa37sbRo6+hpThze(rrkh;bd^zskxRYbJtrjsO2__fYW(fl%$r+Y~Xr!}`L z|J+%186?S<|Nnpk|K}*U4k~-m8f=~ht+}Xqx0mC8njBy6<03@H>ERSOvvSSy!&li) z|7Tz}+{kl(L7d-wNoKxK&V^?ODMr<9VcWOl{vU3h-2*+qsqo6k3jg%Zq~Z+a%oQ8i zy6lbQr%=E4D~jvXYU;eq^hmAX-<^POj{FCm=9&O|y`Z1&#e`-8Qi@A`S4%`3vns*% zL`_dXtxr#p55C zBXq6iW*uZaD-Ix(!7grciddl)td{!X_Xb&A{I#lUMRnXN0^Qzwc$AlIM{-D`iN-&c zFG>y~$;STa>_~ft4m+M`UhNCGN7v!5OdjRu&0U_pRlO%$z44s=wiG!0zdpUuuMI$N zj(mWOBCAPmd&HUQ@JPzbe$H!&Eka%Qs3^PN<|?x2@JSu8(=NibuQ|XA0gsMH;C8go z{?5+nb)`p1y!vg{QMgmLL!Vgg+sls;$l@-w`;BaUw=N}%NkanM?b3n#frQ4}=j>4d zKqRGU4)s`{paIp3{b<;WC8e|Ix<+VIbkPCmsisl+;U4X$JCesGR^1=JyV|jHTbtD8d+#aBq;wUUpVRrCFBeJl{)VC{_&!|JI8Zecsu$jC&VPz@ z1vKP2DnEfsATDVEx6U$W4ZZj-!9zc5Jv73!+_`~R?C43*iOKGMp5k>`UKaasIr&zW z`hrfea$zTWoyd8Zo5kK0>EQ})DR{|q!0UDgd_3_V*2BMsYv&;4*7OGhVKXmYcNC27 z0>=p!3%I%(=J~5ew`^RgW0cN-vqhA+By0s#(IlnZ%x{NcE0^=;Jins@EuAP&d?agL z1OK`xazEO5H`(JDS5xfK0-vT4!cxkTZa*K02NfZ+ScH1j96BsW^wT}Y`iz?`ec~vk zZv7qWCQ}qzsy?<(dZC6JHhzLBP6Y`R$AjI-Y}d%Ejrys-k(4y=4F37#fLC5&dIoOm zLc)J9c*%>ZTTm@z$L@hB0@z67OB8zR5Wi}5zLKxieCL-csp3#j90<+xCXRL@STVAP zZ&TTA1+?Qr*&y+}nysrJ1?B)tr7qyqo?yYZMbU~Uz?G<76w&!e(RXML2Nl=((DUA< z^DbnVUodGjCw_j)Sz()Q&!#R|^WKf|C9Odlug@$`vqCapcR=CpK=elOdYSfIK^U_Zh2tP}^`o2MTGh@2sgw8BM zN2;bhTQj}~bRnO(TJY*`C+E0(oP@P4aF68?3$VWg(Mo$i!1K7-Cvf`_b?{_`_>vnzg10*`K&%D zV=ef^O^h4Zd;W7XeNJLwoemA|`JlCtbRUY(?gk_+sK$)?gCH`BXsYP6 zASjKAvq_GLeS4{{j1+*=J`Jdk>FmwpV6#byRSq$D;{xh*F3lN=YEmy{plOF40L1no zsNUq+pQ*WJ9X>u1;V{OVW$`JsilrB9Fafx{^jK`me1LXdAH!)~x;q*N6o-fRPw`(A zujkXkweTW=0JyKGeLZ;qxCc?CJ)&^6!mT#H>?BnzNwoVpZL#q~NXzI`t3|6Qu5{IF zCxNY=wwW zS^sVhP%0H}L^}hkZa3Vd)ZsDm?9~!`Ir>TkO$p+A5`23b;yv_U5kzq!ig__D<4yA^ zb|KeqDM>)KhysaxCpL>nU+=+;!z_9m_zT8ws2zJ@p_y-6e(Sd5CEXs|5Ec!zGY2-R zERWUCU+nfl2Q6T=N%6I^YL{BN zsQKlc;m>qm*nTuF%9Yq8Pt(lG^zuthSk&J%{N-*frqbP6H ze}ovzXy6F{%^t!w33gQ~LWllW;dAinY;1n#R<7Bo{(=mr#OB+A?kBf%mh)7(H{jZ7 zD1UlkL#jDY$!X*Lg0&6LTA7`$I8 z9d$uUMUw-LDbyt1EW=SjgGgJ3#y+xLP zIK<8@di^Mi-L7`5$UN^;C{iBbuUH^)Ov*_A%X5|)%Y;Q>@<3gyUF>^NbD^FjnArM& zIT_`oh3;$rtG}_?pr`G_)I--vSJ*>!xaHpbv5UzsCYnF?M6R8rsQyE6*9w5!q+A@M zzV;_UlyOO?4)JA^pJ#h%T-uWpW zlK#64{er}I@pu`=W{Bs!Xv)%k2s^Tg{|;!PiDr&Ow36xy<|r1T6YZpfc&96hrT|ou zt%Y+7>56~lMY8cQ=HBHY^6sh&;I%KKep`sJf-$_(GW|x1;2D6vs#0y3i;oZ za-^jCSyGR(D=4xm`pQxxE;(n~9Tb{5^CiTDYSm|a%y(nlUHks}v1f5aR}2V-8T3q3 z_N=I)_A6_*Ne%w2B;$vdLOJJfSu)ipS3TE~pfs20_bPPx_Q&9C!siYeF@6B@~5OLQo#YJPK&408aaDr&o=%a7r^b?U+6?$v(bk)zoW?iL?-JQ z7Kt-`i}%V=#)qzOvbz4NUGstMjKLgeJ+=O;oS+5442qGVxvATasvx~R$Q(cI8PAw_{PfH-r1E8Q(x`LA26@6WEUau@Y`V<~Bf*r; z#*f0`6tnRK65ZbA5`AWX|zIe>@huDr}(bM|U|kPyJlOI@`R` z_F?QQfH$Rh^_rRnu$D9EoTR^0>ZIVRmA9K;1qp1z!$`b&?d1zlo!}O~9E4d*;`Z$- zK7^7}WXfC||L~mxG0)ggwHTGXZsT?nC7=DRVDexoWO=RUE@q`8eQ=-i;=G_$4PC2u zM7Pg>$B31+aV-Ro5ZN7&-z?<;bhez1>?oA^iA`O8k;Cg95rV2^f+qk|_D605UZNK& z*TRCfFw@AfmPbucfK32QcwR9dk8%$X$mSJAjY}x0xq0A`Q|SdX8A9IP__uX^932Q5 zML{T_wnKPmy$4CrmQcoUrs=(5ahc;|I_{T+Z7TUD6B8U+lp#c5w%hcki7V|bIAThp zfX&PDjq}z{+6uw`W2+6`Ku5i--kGd!Q2l$j*QMkXVrjKpo)b;a#0}+g_CHgtg7UPCIlOKsk?|XjEMr2s{%}B{ogB ztX%T~eLEE|(D%Nre77uCPSHR}-Q+zb6-`t?tW}y~x@L(Nh-+4+q^Qhm@d}Zp36&w@ z1qy^DUZQfPh~L-F`F;NR{LcBE^ZS=$=Xj3x^!a=|?)TeGEsJ6CTncXY#n++93}ges z!k)w`gzXG-EQpd@*xUAYS-6oV-G@-j#^JLwaRrWK|HY5SGMju!nVe5&pr=8Y!Vt+l zxkZd4B9KipEOH@^ZRgrIzVhm1K|FF|BS(Q_M!88mDkBds}STk49CDO<)Tg&+FvI(9eT^zD!<|CpQf ztIz2hHtEpE-rsMcX#7<-&sXL^L5Tj5Ot#$Ld$95EDfGXYCahLr)tVW$E>kqiYA^aL zoZ;-!w6?eckNsH;n^+$y*9u+lvF9R*Oz>%vj@w=3BR6kx(6sWms@QjvW%T66zJYJx zl}7Pm>;quw+B3mD81Nx+Q+3|P9=MI6-~9nU_iE9*<`fhzKI)u*>w&nQD$LrrsIU8{Y$!|=@c zw!E~H*U3}WKRW3)6^B|qo);!5;y|b0R0#hw#w$iG$Gym@3Rl)2aZQa_R2(YwuuHsp zHEAQ(5Ab&}uDt&!;o7QV>+=8+d+T?aR5re;f3x)1R{b^bELxVzro%D6u(}6|u471U zihuT4L$or+H>SETXzX@oU_ttO+}Dg%lKLlL`8Nn;{Bqjf2-yB&>2RabIIo@iMEi1= zrD!m9uo`8JRA0ZCgLcIc-X(fr-6B1F=O=vD+nK~F{@JUlC;o2yZ|#(ASBd27>?k0@ zn@*gN#Sb|z zTu}boUna-bqm-LBBK?C6{j=dy$yN(jLidriu^+)yjLekj8^B%0GO>B~t|Jzp!mJ)& zCamsr2{g5J22s-lNt2c+FAeWCKMbubU=jM^@Qz4ZqI%7Ti8`P;S2$eyE*>#S6=}J znvW{ehKEl+otmDsK`_`998j5{OUMb1uP5>F?XZ-i!O!z0?})z9!2( ziY9Clg$9hTk`t+Z`PPihBwZ)eZ*Rzc1Zhdb@+Yb0$w_z@O70}RcXc=v7{h!CZ(FOg zr*?y=2gvJG@AAzd9tgYD4BP_9&7&-%zoazg_$YwiYOXIR`E$t6wud!1&ab7B~e%2PVy18>M-Z z{x+PMRy%%}f0DRmW@L8=liPl_VqI5vxZ+jRYSF!#G@|HmHk}#ojM^6CR1Z-jCb5|Z zbI;8Cyc`G3mn|I>ruTSs2YJnx8nk~(n+nAFtsG{e-wXGz>`#1XA4~jlP@*ZCxgu?C zlDCZirz1sBwNHYUA!y&^gvEHm(VEp^C3toCryanZgcs%INHQ-4s4Jp; zSqqM0mVX@GQH2acJ+%l8z{eIyC7)~hiuVqGlyg$LIFFtI#GtjuoS@O16 z``dP6itUN#%}xHC!Kks_SgY(NwB$7Aa3&WoZk3+5*C1%0b$eYTzJ6~*;%RP~HNN3!*AjOiXeQzzBfXOb6zZ=vis4qA zZWDC9ueP&<3&(5|GCy`4{O2u^%pyjBc5Rs(DDH{BKa+5uz{IcLMAO%1KCrNWJdgV* zH&LqE%R?QYJWN~9y#ywWn^k32Y$M-{aDKsY52n|&C2 zpLSt^Gh{mtH^@b!l!{aK2<`Ig==ZpvoD$tlSDqjh%=*7a4Lth2otgmk8g*ZR3)vTq z+yE8^PJo)gEuchFIs1GkUGLIwFBk+5raB1UA-BsrX~uJn^V(#;;qBweFYeC*6pk=* zH}|h^nBHymXZ+{eTFkX^Qw9iFP=~aT;g(-s-5Ndxv5r0)*|YX_)jbPleUhw#%keF8 zYfXk?v0nO$!f_erPfVfrv>nW|(?qb~z-BTAi|S0c>pOQHgco_G{G3)&lp%^@K}97H zJUPF*;UOyypb%KYq7aQFr{2G=#F8b?8tDv9<|h(!^#1+jZEg_wxbzB1nR#$7A8`@t z66E>bd&)=dCd>__Yf@bUjeIy;IF60t9(KyMaDB-?L>Krf?^KbZ4NKfK@HJPG6RE8T zB7S=o8KWgz_YmI7+FJ#I&7+*Z5WxoZ z54%uJ8G*%UW(pbSSDmz!yvDry8nJ%!wNh3_^rSEc+se*#5V(hJ0*@9CfX?o(j2C4d zMf>F|z_N1C^H6h-mbV|DfA@!D*^KY-x_2tIFcd8h2Vkn>3fxnkk7wH$6}far>==-%us z@95J_eLsZrL-q7myaTkH#BImjj{p8w{O-YWooVz_vbUZ2<(t}TfP(U?mJf+i)t);q zS`A#`{@^CgRbpO5uz?~9G2~VM%?1tk8Gpll&Amn@q;jx!zRo**)5`Iv``ebLd^-RAvl&8PAI8!0a$FF zpH{p6L2mbL<#26^!Sr;_>I;_V?C&|a5EN9Pe9<^nEtkOa&6>MNjjVVb(-gD8W^0GIX6QFf%n8N zxKP;;=1&3PXvyD_fSsR&%25mKR9H!y_%Fj}FZQrJ@v$rIAAxdGJhD#(`51Ch;{$j4 zA{Wgx8UFDS>$^qDmv?sN2~BC+zKQn2hjm@*_0BzE)|v*;WY;+8mP6%4(yPuvK%fnZ znNyD=)RoQ7R!>DgYZY_nTY9kS$CIrIgc5x7TOL2+fX21p6{EV1RnQi2u%{aZnpwD! z{>C{`LHJ(8FL4dh^!p5lJom84#5D*fyGyW1$I&GFg~a&h=x&*SCNx9b!}^6l%e)KR z_&C>{35wV^>2*0Uc}CckC}uW0=*DTY$Q;xRC6c~IgZAd3YXb0wE~VdYd#N`wo#Vnx zaSH4jPhWIQeu1sPJy^k$q1by-we4M$gDw85N5c>A#(=#$ zQ&QhVja!9uOIE5yDS~R*(vKOm;U4|s>fBkT7V_4y_`zgEmeaf@1}HACn{I#yU3G)A z0r98ooqwL$OqxX?bhm+`%j3guxfO4h8y=!_^z3; zLHj-6`sID$O{pB&SJ`NZkXz}ww(^q3w)g~bP16@t>(M=_U;VtBXaj_Gc}LP#{Jy_? z3S2QWbh6bj!IGx3XFr}M{;C)Ei*B)z7~~4NBPa@>x+jXpOTpy!zf^p}0LJ|Bk?O79 zv6Y56-*{&bwpn((K|0^y;(JNH{P*GO9598expm=rhb6&Uulg$yGi<(q+%JhOzPGbj zer=1p&{1;mwSyhe9|Ki%?CQ1yDn3u^cOmfG2~aNB-p=Wi)yYe6-6!RAycO0h-U(ty_a|W|?MP!rPqE3z^WF zS}=VzNj&hIaKX`rlo6%=b7$82f$d^A8;!t;`-5-e}(4>+PRz>x3QVFz+WJ z!a=FqZE?u_?51l0p4?0zI$Kxd65KAPd7BSjhO0MO{D8cBe*Tdo6b$go;=jo6D+&w- z>)T|lAWDid1Zg5g!nIiSwf&VS4pleMAIj<>ESddIph<^gR$f;2e(hk2#X72fIh6+d zaT@2$*65OC4rD*oQMM%uG@ci7@*uZ*&Yw)C(>J3zw#p)%VxqGeHz(IZ?`B|!>PE@*yUaFUqM z{86NeTfT0+em7?%#q~KY;Bm{Q_j{qG9dqHApkN#^`!%zxK~rFMk=!Kmvq1p^|C02f z%p#=423CC+#JjjS4U6xr;GH)7E3pF0^^@PGGRqh}?Nsqal zbH+s}-#=z%T49qor+WHTe@lvm|?=Z~sgK!vPxU4`b`;?IYn9xvqo;XEvnExZ4Z~~J_E+dRxE=-JbVHSaN_7#5?721b$|lw zMd6irN0rF^f5P*R#sbmQ+`yWEPUCz! z$Ged5R{;k(WT{lC@;w4G7dZPIF--387Z>-x9K2H#V2ZtTPw|y*jFk!<$q8IuvpP-X z)D~1DHDP4mskOiAT%-3qCYr+P%}j3C=!-RJ_za`;@iy&DBT2|Y8pO3B%W*N5@EQHq zbiGBm$iA?08~5X6W>%$7{C~j8wKg;1pDl!hSYwzkw$dzBps|HpN-cT}Ty4M}M}PoP z(f(e%q-pS?Nymkk`6!G2+Z=fzk2LuGQiG{s8J4waS+?b?j?f%QWY2Ro#}GDC(cWNZ z_8A@s7Vbq;l4kq!PADYfg-STOL^WX)=Vi!lzAzU}wOILF!M*kdGUiYhI5f;G6Y~p& z**<(Pjrg%;(0UbbrTr}Ee60)0D_o1+JYP6jpH283fq?=&Zhg^?;j0JA>6!medcgmL z0RA7~KvwTZ-I`Ox%0F}?-^{$#>}{WRoC|oeVoCHOxNjbqsa{zcIfkvAUH5C{R8)?Q z?nUBM*VKK_;xn#-QpS!IjC{qj++)_08AD_6T(txK>gnU?MhYgU(23Ulj(K*bq%>tp zTmNG0mS5-is(3TD$zRcs)0lRFBUW58%XKt$izl<{A9eKqkfl4#ZkC7kv_&tCFpk<` zN@U|h3KS|B(`cha!qR_p6CxAv>yI#&mOUQC87!C6of%f0iK#eBpML&aAtkuD899bw(L6UNBVI>(uIURHPPbqmiBA7M-=FlEgZ4{*Z(N)ty_-E@|M(@CD`F7~j1D0AvH2Bn|zP zBhJVUbWaj~meu~=Fd>xNK8@TV!J0C?ax8PIJG?*c!szqOSh|-AGfM^4^++4fN_ENa)89V}m=$t15U}e)ssIGY8z~q8*j1p3> zAD~@cZUc@#U}%WB(7`6CDD zmI1_6d}Xi3gFjUcC9%TIl_Nq8m0^wJHl^Ou;uuQD#Ihc^% zm|pjA)Cx@0lx)igY?TI@pige)naOP(Gu?$Xj@$UG1=w=zR|WS_FUJOZ`h$RoNjqhD z4|T-nKEv1$raNKRy>!$JiW*<(Q2>#7^)CRG<>rSO`SqJgdf~J!mxAQJJquG9zn$&3{*ZClDIG5@*pQm}I!V*?S zR43c*h7bUsh@Bgj@C$RYoxNIl zVW8<>n4Q$Gv)pC!%a~)LQhV1^L0kM3+}hhj4B0p!*36@#IH_wjCxU^}<1=$5X^HUF z#rly0=xwP2Gn@cQEY>{oQr0#4>-Q#D=vGl#pIE*M-^W^ z_3b|lZrS<=7@6q@s2MF!nzV_#ul++Q{^7-E|Hqoph`{b@Kp3u8jJB#hvYGx;2DRc^ z3MH|`2+bwEDtQa{7gBpB78S}#yIRYgXukDkPtlslth+*;{NQ~4x@2~(H_WET4i13C z;2Hf52uF*0H_a$zcZJ2%E0mR|RlF#0tS;_mBwclr3%7{Ui-#U&wGT%Gr$m8Z<9Pb@ zbduHHa{fuMAL&hI5bt%$V_(@6337GewDZ*5RG0B~T}e^p&f*vQXNHpD`?BypZxNN` zt6mO1hZ@UoCC)Uga!+M z=R$X6?;M0>OlX2NW{P`TOiFG{3%ql<9K!iD(EPYH7WYt^BtLCZ*ayuYS(bcem}|ebWBV1 zFma7C62^{NQVi|s>6d&3Cs9%gdRS0YfLbdPnK2xJb&Mj_%JL-9fAKikWSIioR+DoN z2!hhs9}`W<*Ig*{pwCTihL^?9ZYE}k=Y>kmfv0x9Ae%xcS(${FnfjUqLEUS;U-u|3 zo%ifrO-+0(kDB~!U99+3WS6E?Ie8P;r7T$}CA|d8$Gte*?T@`pckwS+qk81C7n)WA z`?>AFMMXSqreIkH+Kl{Zmmt@CpU{!6SFsq?W%`#7*boeDxSTm9Wes!3zH8-isDPIz z?j>B}NKY0Ubmab=>aE|fHFZ!NwP*M=vHCN9Y#-`%hhk zI2pou*UA#h>+Jp1=OG70&~W-B4*Fmh3`fHGQbA+E|6!d%IhP-$)7pV=<8$g6FB8y0 zHM|j2^la9)8x>>B1tHPWR%nNAoX4wy3aVFo%Tp3pq>FCKg$+oV+Ksqu&d;H(#O()l zW@g(YXz!84^*o!Iv4erGsi2_16KK4(Qzy%RvWNBAR#vWOHEI->8v}5(^+U=IJeVq? z-)7onNQDoU@)=`upNfDEzHP9ECCzW`mL~mXDF6uu`)VIP6PdB*vC(&JJ9Qn7l@NpM z&g9k~SCR(c(E0QZoh%zZye4~!vBacK>(lHu9wV;{F47rWqG5WtAYn&wYdQOe`}eKd?)>N9S3k9VDx2rF>TZuh zCUtAGBg$Y|@64<5+Fd7!el~rFD?4g&KESp6pevf}YquSat90jHy9ZG_gUku3e)87e zV&U~gw+be)oN@7}@Di*O;2Mx*Ez4`n8B5>J3GUll#54Xjjfk}e!~8H8!+Y`Dr_~kQ zWLfsK-Rh5Wbgo%Yv&=DWZe=&+0L<{-xPdjKwIm%UV^XeE-1|C)B(zOyUhXRU+1i#L zCF}bMdM;z*=wLyWggDBYO6EK`pNSLNuur6o$G(iU0#HiA3%oO}-n;N#@q+5WnFAem zwXJQ*0MyKunDu&_V-fgKr}EfX&h%|qCj0)_=$Khm3*GFx&(;2>P^w2wS~%obBI_b1 z{-;U1@ga<3@{%Ve?E((S5uze16yJ4NG;ui>wO4X}E)=06m(=wr#kqR>Nj&VQ1`qe(ut~yD`+1Vp3%t|%4dy}!c=F_PiDV_=dcX-3zWwbU7!6TvV%vFXPnPB+*8tRl zUmgTbqwJJ~IDwMzptXcF5B!tX68$1Ob~WSlGP$4%=&-}|KpElL9k#JzI5qjN2va&P z5B&5f{KLu2k_)y((suv%{>beI!wJh_$6#cy=0D$)u4mNdUc7iTQ8RU8;S1-A#CA@$ z7%BX(C?s=!E5E9B=rj+Wm^^!z+H+;7k{_f>l+(m8dV>tf7#RUM_V_CRep2u3$8cI^ z%-hqgMb*fWSzx9&33Jz2KK*R?rkIVV^O>LCoi&o?%PCs^lwEr z^}DOg?a3O^Ux0MF_VxZ2*k1 z$tFmPpu=@^a?hO?=n+VViJje7;~ng--TWT3th)735R5zIH4=3l^e;aF5>6uIwWmccD_I7l&c>U-L7)C zYlf6x2**HDElJn$>4(z%r!7WG*uKj*7H+!o=5B%mX$uz%1$F}M_tVjNGooXnsFa!> z<&{h!Ml*Y*eP{kq_Mwk{_?1gXf4F<&!&jfBgkw5ds9D`HL&!bPl zG+&Q$pJC)fyAb#Ms-xw|$O8xfiSATB?2}M(&q8GTW(fk?S7Z;{90LIFv~=BFc03YU z5_bqB+SGH`q%|UK^m_!eE!n5YjZ1!RkMvmUI)x_dqvd^U(L?(n&HHaw!VGWCM%O3M zo|fbHI`5$^mMzrjRw-kr!LV?B^s5R@MN3>L=1s65{<5pbm%6zLR+k$Vr<-H?XUZ2- znV`^4i%$8PXxa#+asiAEoE|M}rFI@|S$|PvUS;}rd1DTlP_mXI7)-dYx&W%|=~#+G z9CbZvk?D?gYOq>f8udIs4(t@R=6>gdr?1_CfngnxltY+{-{!}62$*qB^~jNMTeR*7 z^dO~(W-v5WmXM4UiW3dO{{uIhM5Vc}A{_ER8$MwLLoVXf_X!6$ds+#aX89+r)=DdL zW!-742Y7jk+t)~CR2068-wDr}bVT4rzLt#4PaU$vG>$|mkuy>6alm+&em!!`%KYqe zAy$k`^jx&97!ILrwQqH%r`FwLw`DH%f3p^fZhpdfdrQ=F82_f=ytypD3Vr;&ObAEI zHokZD19YiZ#f(wAG%wm_uoWk0sMI_qnP-`p`V((-q26Qff^B!e8_f?Fl5&j*P*<}j zef=OJe?59Wmo@@W%6nJ}!VfXpC0;v{zb$KQ;mY5$v$_nULZOd+fHLk+*A9zZTfn-t6T2cIT$^rcDxfyBzp7=YL3X=m-~Kl6L(uUr z_k4WkQ^#GoGXwEGcR?uPjYD~VeChM;-Q$}dbglLe{P)ta(x4_B-%XU-iR%X3LhhPS@o|d^+m-6p{)7 z#Z*^nO1b2f752aFd~gVL6IBplA#qwA!;qv`pU0j|Sn;OU6&_G#1V$}=barH<$yaz8 z>bAH5sFMROFQ>l?-3dVJ%c`u&UY+ER{f`!`%Ik}V_4+r#?^W=e!?5)ldK|Cfhb>hS z?ni2;k%VJb*4a)g=O*m9n&&!Y{Fe5Es_u~yZ9pdR(moU2!S`n?eDk0Ho~hZ(5!khs zNDu=XQ>!^q4vORe<>BJO*@r>PbgE+it{m@R!PR#CIaX>h5p&i;gbHTEXEs@P4xWB9 zNlY-5)p%TToSG1xhFxv?DG}os`<&NZ&8i8#C+urk(^e$b=?wJV|FPZwFMYuB-v9N1 z=?6=FW1v1a@bL1^SMSxj?*q18WNG8_Dvc|F>w~8D!oaOLIsSJ$ZQJIzfqn&mij|n%S6vWS z1M7Y$8&{2@po@N#RrMOD2^1r`6{2twhnTfPki%tWP$%Gb7e^Wysq z9aPZJ{vae1!VPl(&XBje&=<+!IAKUny3D+?K+sM~pG92%Rcrt>k^n5n!s)G>9W0IMSMx&NS2-1d(>};{y%wrntoGG> z9*8U0`d1f8ytR%xQ!8kvx7gtiwYDh^qA%bo+UJF!7Tcw0d95zC+1-u%jDcGEaa%pk zed`g>f=ZJCFNjgeS7;v@H^ekD5;n3(Ie4B|7xqh!2ivZ=3S#;Q2z}?qaA5Cc#hv4A zj5-r#5_r=LjGRR$bOSFm+L^q_mzzR+tte_6~Df)wHF|v z0UX&t&Gh<-9EIVlZgT(ZFS&*0F;ZCBVb{welR0HurOv?~kJl}yDhxQ;`Hc~aAlbN` zs;)%!WuP-(vNKDl;3BZb>lhpbhuVH#nn2fX#xcRE&kk7FN>c(f!-lmd;@dy(QIA#k z+>E#vdkLBGzTB@n`8DN~*$iC{oDswV%7q=aYtk$Vcx!(?Gp8bt)OB)Ot=-rD0{GEZ zS(%(WH!;kob)NZ=^>O`wb$~d1rMxoak{ar{74Ti`f9B>Ls<#!mW{8cUbKi|+68!epC@LNJ=&y| zbE|h&J-+NlA>lh;T-_2?b?7y7(J)|7iepH#m0l4CuPS!Qw1zkLA9gU?iKZEFiq_EuPmT^m%y2%FK7P0*sZr3M} zKFW5j&uEl0QCiZHiX$Y_EOxe)ZEFwE{-jJjCRnB#@o{$j5OGb+KU&{DMs2g&`A$9J zV9!1R(CFa%ZBT^u8_oUCTFas+HD10b!y3iK^v@D-gVQ86#j_v6U<5vIl-}{HJU2dq zCi&TAhPLYO-E`1*>hNt3*SccmoG;*QB0SB-UvpR~`9@7r`*l)D)2}0(?ARP1ON_oD zzK{t;YjOeTbOo40sY@Xm-Vn#$Po73H)mEK;Na$V7{^gWQ7E5mY`8Jw(fyK_qR!?pC zJHrfAR*4FcKmNF+`C(u_o8*|F;dhqjh|KRZfH~$=Mr1D!WE(ZUdR+6MM?x5!-OM}f zr9*5$7++&{Jo>$jW2~|$OI`~y^QT)O#_@W|*3gNdsb~U!IyAE}EsZaxf+|l0Hs+ki z+&Z=VA)g8^sU z#C>hwDe4$sW^4`|^rMe(&EYR2_Br>oxdEIMc4^;9B0L!phonOWdKvcU%TPyG^R~^e zo?a=JPsm#s!b7m8C%Wx7^?`4kTVS|ypcR^bWnQath5I5elc9*HcMUIVbW3xWp=w#} zNiQdfTRP`K)n|+428sOI#?|?TtobLg#>e{e8Kj$aPnQ{bH%Qb#g0b|xKT>T} zJ>D4#>bd?pcO~VY*%l~zq!zvDw6l>1Y;9FcGWP+;6q2?(+^j9Ya9((xHbd+cfceW$ox)K3bmZ=~$L8#Vby-oDd10D{w`9cgWb zP&k$E3M;`#uc>0vilq-5qO!^Pg7-y1LA6osT5puWZaPq^si&9k`K1CNU2lNQ)3)m3 zNX)Eb6E2#wo)u_i?_jDP@6a^yzWrw-$0XIhN!)nM ziOTF}JQ*1^Kyjg7si;%oL16@ptF5R_?Zuqt0b!#1{8BmP!hwxnKPG{R%C~JIVx2WR!;Nign2RlDwH;D5?)s&m&Oj;Q8 zAx)FbQ_$|#Sow^ZS<=DN5eLdw77@rUZXL?CB|YhDhLd=h+AQ4KWVqN&e>J1EK3i!J zZ+3SY4Uz#B9y78{uI!k_!yIzi%ALjPWCyw3*b% z{4;Majfe^Cjero>=GzA`*AhsY`IkXVQ5-P5imis9_BSnxon?>GTvuLj=?A*+z=5@f zC7KQIITB_$`NGLuFqemR?eDoxnr|!R1vJfJsve zF;0Uh=yYhf)_#W>c!bN?VrKG~$drPr@!jh+)urSGdBVrUEb@ZjDpw?sJYpS&5z~$; z&yY#(lXM%L@YDTZ`rO5pfUDzBkjg31bIgyXyc6j^Pcx5yZMeIx{dHOr;gY1eT3C&_ z9?xTpsa!7RZxSo+G_Ic*`I278M6HR1FAC(;KqL1&wWtmq+dFHxV>)SA&)Pm|JZc!} z*<4uhAEw_M$%PB1cU{y>4u_&`TF*(gSBPUNl`H@It@r{C&+UIZGm;mAh25=y`LI5_MzYz#F^q$6Q_GVR< zjc&R8`^7A&LxPs-%KwiZ^$Cgn<>tMkbch;&70mqdiu(zqalma>JDNczAl-y6C$WAA z()OO?N1uMv3KF@Z_;~Gg_qOM&22~KvgXpDCGNwCwZGTd-y0OSVmqDczLfy0cg_zbd z7u%yjIKJ?=5x!Mn#NZ?l4as{QVtDr=VTN@@_3)_0DbVzU9~frs2WS!m4oDWI8gpom zU82T0?7;NCtZte9u09##6qMOaxpN|!_s~u-_scn&;!BKn?tTKs2~I?E8IMZ)!iArm zb%|OekB%k&C>6m`83b3o&=%Ycp#ajzC>Im=*(s?GfRCw$3mn+p3PEw;(Lh^h*ZLg3 z61%B|l^kEy?3D+y)nse$o++rs-!Y2BLfit(xyBS-W^am?!r%T>8j)fNK1s*p{I=HW zrrVKXhDD(W!MYK{N{kd9M>+R|X5ux6Z&mGvquNv>d{MjK;F)*P96$m6xWrA!HM0P0 zxWI_Rr}oVLtc?D5Ogo^n{Hx|Qt^sg^q194|eEhEo++j|RrE7WQ%;R*WD4B`24Q^iy z>xzn!$-H^w`d>x=B+fmtB-={gB}qD9!5Q9jFSISKBDc7F?=I1*wl|?Zo^i4}_J*1L zO(D@8-z+^h&B8~n9nI>aC(6~)iYFpQ%A5Aeg<_0>qe=N3-6GND2@xrcoNE^1 z{z*0lcs0v!O2O4eQj#BBo6hOwE@8R57LGg&EbqLPTMY#Bs4jl&@wJB1eFY@Esrezp z!0Vif`(pSgrbXKT?>!%!IdJglR2$6_$zAR9j#0`OHzhs=GQ!-ru7LYKL;lXim9AAi zq6SvlVuj(`QrT2H$mVRY|3XabZ*6%DlUwowOy=b! zol~(3%9<5VHj-Qj{t9_akrN%5CU=OwnK6@{+T%?NwLq8!-UJu}6PyXnU)`RA8N1nQ zC+GlzMAI06Y1ey=u`Upf`oV)d$G!Jt(^yxX2B(HZ0y$vqRSFgNdr16|w(ghlGdh~I z?B@*G_tf<{oVh)pZ&i;P=X?q{Wla)_;+MTtSMt^$D@ZB8;;oE^mvJZFfgI@@gql5G zn8v*?=Z5QZlHz)&oul5K2g5O1dSyjvNt~tSQ!#hf>9==^9yoPtqp#8!Q5rq4fOkyR z!7N^}2XW8pI>x!7`zqTv);|n%-Wm((j{PsX2hwkT{@t_I;Z;D;{Jd*bIsnviNsCrD zbsdwXRw3vHHeTlHMJIfU*x7bjOCMLiwJbncLJRRUH;ra}XMN>XH=Mjc>`7P8e)rB; zOjWv-PYBb#%h1#6HlL4e#Bna6S5>I}=*`lVx5JmVM_~2zn&{ex6%SfFIE{(Sf{s!{nc5|m?u)5hTJ#p`80PPV znCCt(LigrCs09gOU!SZ2GE6(iSy>&A1(!BfPwz+*_htl<<;3o!c_U=;qeSMz;UrSK z3G?9aVC#WnFL;vP&pz&Y^qA9|m~x%5Pkt-@>ITq}_WkTwmQ(&dXD}%o6lM|mFPl@| z1t@FfPp{-YVUOGJxx!m)4^??Ejt6P{UXBNrprI-JzsmApIj%|ABE$w zGE~5g1nG|f*#CKV_E9()cU%Ww>cm{)*z<0cG*K$AlD{_%uufpEar@vN`&|LNT8g!2 zZ(U#}<}drFEQ9ctQN=(p7c}ZoU1_|DJ8^DhFP}XRcPwA`2}Cw*{~{=F^cCOTA?-Pv zso(32NPT9nn*%4Z5)-{ZujTuI2HCe{)^QTwKvOh(v)DyW@6}Htl(#SxOvbR`2vZU_x%b~efm|q;k9*DMRP4PYZ?h<~3Fmi~`x_PdFFW1y{_JXko%qks2DbKRGy-y~iWCs&Hb*enwKS`uq2euFJ2|wc+(i88P ze7(Rz!uH0ne&R*{Fxa;UVGY@us zsqGItT5h%n5L5Lvf|w|eYzpu1V4Zj|I_Yo_@9@csB|pXw4`xNTbl(%RFjWvAsJFIa zN2Uk?ekjVV{4!kc+o0{kRV^rw)pN~{CviXAN>?UnCE%fhXkcYM&rM4+ifMb<&7L|X z$5CaN=(jmrr+QYrshlAZz1<*-Qbx0Xr;;TnPm74{%(*yu?G-PKiX0XmUcdLqc zdxZ>)95L$+aPRzd|1I(fbI}+zFaX#`gu#P!H#%uivSCTZD*!H(`8w)t6T3nBqaziS+V=G!NBcMBU)ph7#(B${5-`8tf&CI8Ael9(E|6 z9ZMvrP5-~%m=p}wNdoYc{v!Im( zXb-pi3tHRZ4h-&zZD#SYF^}N4C#c&VJv;LUh$$xa%1#A?zg>M(0vxo@`@in!I_{ZYUf`N_|)z1JZfC*#Hns;oEZT+iM%J;TAd!1c=hEWK8=m;~v6 zCI5vPt9(qLyhvcWT07<2vR8?g)4*G&wMpnOy(qia>7AgBR>fQ+asQMT*pIc_U8RZp zJ7?_TNYQV9d{Qo!u+vRzvrR9nKzS^r`t~)R%Tvrl^F+=5#)GziEAaa&NO%)%c(%WrpVJl-lKA-y953!pm$r9u1Ya|hx6Bfl)uoGnbrKeZx+chOykTT7DCE6Oi< zDqAi6mmd+t-`+U&$^WnbqPTZ}*A+8B)o0-#`3Lc7=Bl`TbJd3zVzH>6Z8>C}PMn#o zbwXxz1rRMY#_;7^vmim`g{~A5Nwxk(2GgK=W|rDZsf#U%yB<69aD;w0(R>c}?$=f% zR`FjEqfB^zR&4D{^6Ct&O!ni@1eSmn{^lvZ_{t-Wm04$dKd%w^|AHIX<%r*UywKDB zdgKpld^$0+=85i0+Q#FFre-DB;^^K$*BQ2_*?TgKjq3l|?pH{kDB(}sg7Af|P zrU*%du~BYsS*Uy4W&j9`WdF!GyE*pCr@M~AJmFfzF3^U0;U)Y`!Djw&iVcRlLs3Yuj$Cvf4t)7B_j`(+;2B zzVPr`{QQ0(g)OkGK+h=tK+MP8o)JC1a+M%>!rKjdzg9d+UW;Yeb#!6Ci^2lGEU(nr z<{L^5?f?$4wxkkftCTfeI37Gb% zmyfm%nf16gB;l$j&(Yk}wfYo6=F`n4xY=ipSFTw?G|JnFV=4h%N?Ud!=gA)3QM zAfSYc(IN8|lwr(AmSC$`;!l$u2I1xB>)ME;iaEfuYm0k#3sHWAYSW#PzLo5)Gj$E3 z>!v{ku_5(q2{WWH<8*HLA3cPR9n6RX;}U#UKCFTa4D91D`AK-1;`U7u!jtn1w=Qx- zTpq#Y^qtX*H)I*dr=^4MI{#4!RzGEU`y*0o4+hp$s#ug$_WLv2=dDZ?ETu2qbpA@a zF~SYVm`!vtjceh;YTo82=hvR_F1fftV>DQ}&3^w!yfeyuaQj84Rh(%`M3&YK7kX1FxarIv7x+V=nTu?5*`Sk^*QN7h40~NfO_^eH8*+1) zR;jt}tYwR*5^nFX(x+}NAnmm>&@>R(k*-;N9b}&=145`)+FbhplU5g~AL!f}eYicFSe?t+xe1rAy7ca;rcJ$ z^woJtMbwZuVrk%vC$&zU+)Wa9QTigGdMV-3-aE}Q%&p;zyi4rsw%6UK0b%iVWvsof z6oIz;e4OAt+NS8oC@^xtMMnsscXUZnZ&Jm0Q-P$LZmk!akdZWh0?@a6flZs$9O5JP zAg~!b=Lw`W^i5PNfiYw z2(4796$l*4p5^q|N<9j+q68!)ER|GF=#wDb%9k!$SJ7*xLn6BmAxM@e!f!g;`+pEp%MO<7m1TYk)?{;;UokCdl7cL{WeD-EQKB$Ne)HvOS=hL@;miM}krY0---DuBM98y)-$4b1zVUor&x zQ^OTbRkm%f&)@y~v+T6k@zs-xn`x)g-NxbBh7SoOiT=TbyCUogF8>j?xCz70f|&pi zZ$@ML?Kc^%3-m?wj`QwP($HXLe9i@Mc9{(KGVB00EtYtr|9IlY1b}p&;a>xx=_-}p z#j)b(si{pxt({+0YMD1-ZgvY=LzvC^41<*B=tNjh_;KDH48O`H$htjk{W5@Wj*<-O zNtVF-nS^oUF|Q#mXt=N2sZ|$3m2+o-WYl3Cn?oP^Ym^nr8EQ(K{@%Vrad9G|>8-kw z{Hh5l>PNwF$_|~Fdl~JKxF?FFp=}R^lW>}QY*e#dnGvs>9f$QcMcyd`bX!44{jvnW zLu>m8navYN6{kTtgjoU4~+p6e$RH<|sDmE7kRplq1ZaL?Y-qG(O@XV#M2@PlBwe}k`&GUO zyG+-=lSa(2R0pb=VYh!UU1?CzYTin(h_S9X8~D1uFJ74v=0a29bJJN;Y8E6QxJUgek}69j!xwHFc)|sN(o#XYzde z&(ms)%v4uQy zk%oX{CgUzf)~c?pY7q5C6!%HFej-CVM|CVPLNgF+r0W7#5&!NmyVu5QSCt+sq;Rj^ zY$TkbE1dR+m=S+>Y!b6zWwmZ3PdswB`GY7aZ$Gn@*2#(NK~ZQewT&sQ(01(PL+=S^ z0-%eH8GqQw?5<3^%xYiud6wlCQg#;%{Ifm|bz4e*fBbiOUIhy7I64>JQT7D6klw25 z^nGq+w>tN{(_6>I5CbRlw9 znZ#tvlDlZpi_n+ai2R+?+!L5l+{yQK>c|kgAPs|EAH$9lQs?(DQtuZ2gZ7QM`vbZLU6Bm|g!K zM4+FRr3a2Yol^9lsM#kXS&|4j$Pf^tkr$!vdsSoG8FXMI)OscwqHi^w6oVqE!t z?j+Hsc;7i>Q=?xEK>bKtXTEZ-+I-PqMXUAp+k-KNYNnQ58^T%c#7jkzX~xL{XeZMe zU?<(cAcX8X9fN(9;4=G9NAtS&NBnkohB-%ZO-ks!&lpRhyh<;IaK54_e$4SB`#EGY z#9On5oGasfm*ZQPjhfRnhT4WWT1C~6OWm>i@t7~3-p=^LC>jSEm7Sr{Qe-suzO<^= zny5*DD{|2;J_16JnP45j(kX7bQ8pyJh>L0bW7Y229PeTEsjA^>!@BT~Ebb)yvrG0> zhoW7!Tk{-4VY0xnsPfMWg>|8H4HTSKeIQt%0G(P@=6lgCW)@~-#7$_-_%}Upq<_CoSjnNhu2p| z$zv^OF8k{rE?u0_HU+Q-6vDktL$d!6$Ec01+a)-8+3aagJ|(PKv~T~{D(5U3V4x+~ zt{E6b>me3Sh0(Lm&nZW5*~46{F+Q`O zN+!R)+ZB1w>#N_KCmw#~tt@98^7KAav48r@UxRKQ{2WH`Y!uWh?&qw@&0~Hegxc5{ z6)p)WwL=ekCA)udwhzJuadlIM=5X&oEXk$DmhVtU1@A89u*+hXRt8Jg!W1qTk2mz7 z(?O8sZDd)KQJ-|V-hIiy&-1gxt8on*OiK43Cdl*9hsPhu>xJISntqc}Q=9A#9xvF= z&%Bjk@{|WIMOPUx@i14tJeuK;{GiVV*1Uedebf=%S{c;+Vfd;7WF-~6@XdcD|2~ML z3Oi2wu;PBT3x~8IlT3f-=XAQSBmXF~=}y!}Y?v(>b7X(fgRYa|8Ep9x8=&gqBs+NJ z248V-#A`;jwi)ZxSHUD&gd;QGE|d3tf$%fG1f_%Jd|JcMBPBiDIhvSGILABU2+!Vl zS;3DUKvd%^<@an%!nc6EXxnSYfM_S3Wqi* zZr^N@wY^^i9#3@98LmC%*XAa_5fb>0%`GkSM$L4`uP#6i%KPtV@k?bedS@+<)uSk? zVM7qV(!lE8L(Q0fynsXYYKKVw61hIo{c_={Q`{=?-*)LzUVjtZ1D4gzHT# zEe~qt@eU$j%BAFMR`pwPee$#>2zJ~Ht@8$&oy&^(V}YRYK^)J-0ZnMl7x{CvuHg>0 zRmtNEPrNaWj$Ya1t|gM?xDUx~xdPP33>D&s9_gP6A1=QDn*ks&oKq?1B`yva>5c24 z@&g%TUjDX|&&D><*!HqHAm?u$^DtnG*V!o0&?6!g+_Sx}q{nrwhh5mHk&!_;`Cshq zCWz)Vs8IJC16)ywRhLqnohmr;gErY$y6{;aWqI3bu1T$}+nRpqKM~b2C6}~CzPs(P zeazp{)w>Oj(P($}@AoIALhsY6rhm$fPO7nzR`#yFSriqXMf2FXhfGJee#A6tp6?od zKDmxf5V!D3w8`i`{fWMw2$zIb#_fc3>^Anbb9B)eo9z)GKH2jXBS|&O+7H%QX?lhuR<#z3&z?$y&d2vaagb&MQ&;JIM*46GT0!a?S`PlDnsei`Y{V z^^swdnb@fb#oKJ?9b8`o5OS0&SZlF4uL^V48buVgcb0ncYY5;Ei-rWr48JvJdFH|c z+wm=iA9FBgssbjf*n*XQv!TzsC!H!69pRj~sGP9Lu!0AGvao7Pb<`eir|~dt%bJJ6 zu@%zR?6)ul<{uDe|f%+;~l-fR$Y3tlt1;D(9{9z&AvtzQ=Qd0T+(S~k-G(8d)Opbwyz#D$T4#8zl1pRJc)mri9^ ziUvmZ6;c4v-Z3CpG6Xgu;RJb@$FS#_X=2{mD=V)(fi-xN`Xq~gg`Nt*%V$ih51xZB zq&Ls~?C;`91stJy73K^M{j9BIEp*(*ytzf^X_8>ry*KQ2-i?|z;y#lSa&|&u`FPbm zkYtU~N0xKf)fgJXyTeeI(cW_RHZQ)o0soya!nZ=Vwy8R7Q|D_qH*{c3YF ziBL3Z<7me|>X_Zx%Kiw=>Dqkik|j-5JVO|t7)oiN=C+4GM4za3k`CPhClMzHXrKUE{BF@!Y46Z8C6br&MTt4LA5|Ck*h)z` zq>OIvi~GL*ikThTa<~eO^r4^kS)l3or!6S-u8Pk%cmJUIRen;k5$1+gV}{lgV>*M_TfFjN^OmD{%{BSXh3600Tl+Z6 z6g!rCo)^+5ssFJ#K|kvngdDdj#4inDd=%LIuB54&MR!hecmKlxvv7zjJE8Kt*mYAi z6moZo*R{}IF5&uc*T<<`lgGKPqciG1e3$6@-go`$I@mSN=iAz=b*5{&!~Sj!-_#Ml zN2J3r_}#e(XnM4i*PQkm_v0Cwqygb)`-t)gQSWZfVA7>y4`+VnpQ0J$J;Lh3no4yw zw0kz?RBB7Kq>|>99crD{{jPm%TAL}uTxlv@iTr|oKKV5_x6!IU=y?r!J74)ej5EFH6U+TD#slM*0QuZp>`H2R3*e1O zu>${>{rtZE1p40nzZ1!;R$ouO6Yu2VDA6ijwZ%|bfN1CT$*lCZB;vXGH65Y8gxkvL zZLD3Hka?NduBSS#C#=qBIKsGeh>ZOGKnPk-G+$%Qer|Z<5B)pp3V)QNZDLru=lMHj z``X!lj&d*U+>Qgooyu1x^1SE|hC~ zN$*$ha)liwlkk%=?#m5scgDY2$^Y|nXz>BkbnqJN2iDfJ)}W;Ya&+}n&H5=zj+u&0+^2n*&tBX94vKpuR?f)*Z9<^`R0t%lqnVm*bdgjJ&^SIp9kAEwGi1 zJ*xA@r}(7%)FDYePz$uzgdL1V0ZK*eOP~BaCOrb#NbM@OiInPl6ScGhEEE<>1e;>81A!>9FBXn=vC(m6>7w0V>~ZV>-kpC-DC*o9a`;r zMl|DLQ3N<24k^Q)#}Z7dLwD)^8&npQh+D5@Z%%|T#Hdm9?m7+J(Yf;`(RwG~J&mr` zd%>$v5PLzlv;VJSqt%1~aNx>1_wO60M2VPwf9*s^MU=IVQzM@na(p@3muqi%LSpEep{y7Z*V)(c@?d;DW6S`;mgS z)4EEAqYkVD575}<^E?)lZc9oA2Q?iTb$YA*m_Xu05Jh7=X+mudBoAbz+BniYF zwiW#obI8xutot;^aE(MNetb1@);lv&*dn3}G2gEY746&0_iim``p)APe=rLt=TjCj z>eycUzQPzcQB0A%i7;@`0o*}%@`rf8NTv$YfeBNCC>$~~fY?#@ylTe@eq4lQdhWgt z=XAY0%k&(_Mki^Vs1_DpbbFUwZ=5$pZHtf=?Q{nrwH#gKI61EsZ-)aurfa=*?p_fW zHQbPFJOtW_bq&$H5-mO%Jsb|r>%P7`!4>C0@I;toK6-$+zteYWeZ#^n2{(?@AgNNj z-dj<}h5^Tx=JhUSD+RXi2WnV(jC)Sw^M}VNEm%=O$voQ3 z)?N+qW_F&k$3~qrd9m=z^p}5@eRt*yc-8(BRZy%_fRrD;5_!ymdG%*vM|!s67Z=bJAPlfM;AjP1<9_vih_t_Hfo}OACI< zI22GvI1)tYmJOW}{L_?DaSK;xi2+ALUmf+d%ewReT3`)P4D1%*vu?9g`@n}Rs;cck zX6P(@GksL1*Ne3YFGtUC;+_Fknq(TSvt?@bIYzzm^YnvuuYMh5AK-OaS?c+!W{td0 z3zx3J#h+Fi-qqB3Z4w%&{iE&}yX0qiYd`}!@ojQgRQYKv`(@sd;3r4&+Wgw^XW5?=N(QnnPoAo~^8D@$?cb{<}8!LXo7$$HF?(Dnmt@d$176Utz{vUwx z>H6B@qpc4r0a}*4tD@Gh&NJb%Ag;Bt;l;c;>9 zeFs-j@D9GYS;Po1gA~gLhs=Yswv)9}Ef4`dFluqfO}f4g^n={1S3(B`7qDtx_m3%s z{m8|V01!Y@x+b!zV507@IRm`dZk)5EDPA!fsqBuj67nb%W zsI6vBvV;i+I;DUCsX01-^z84JLi_O5YAmlubfE()P+={Tv3AzX>shH;O;<%-Qb^BV z{f|TZ5s)cCFA{xl z_ua$YHnDTlf}S=6NA<#T`!ry^9_7#d$!Z~Zz8$Bs7$*J`t}X3n%ZxUi80!-*#2ty> zM$@$Vd25^GU6cp9EXgRN72S0?)qcEAZ=8f1t~X@Nvi8P0JMfCuUF7B^4DKd_LVumt zTD&Ha7^;4NVnwqT=T{umn4hIM zgU;N*%%fOQNz+qp?*^wJOLRuDIRYf;WxiRYff8X&VOZS(JDuv4FeCs!J$e2YcX&mL z5FA0*w6zXc<+`NDU%t?G{M1bz=GlV=AA~(*@0hYm?z{>^;FgWkQL;EowU{rO{Ik=y z9+<4y|07}eVCQdkZT}pLWnItu@QXT=o3$5S%qQP?LziEmoJ_95sjpNY@4IT(7;*_Y zksP)V4T#`dNC4_$O+HlW^0Wo3th3awjrccNQAY>tOu~Ci>YuLEU!^kqi5#f>7iaXu zW^CUGC3xrD-MiA~CZ0%4f_9nVPP7$>5Jp%Ka5eJo$*0sPV;mTK|J4uouTCxAM6EU) z$S(7y(E@fP9fw{uy7A~8OL-IC<+q8v#tB(aM1fegw!l-FSrpcWU4D$mFudKJs{wk= z-LIH6+Dg4PFK7u?BXR(Uy62%G=$)?3bNwNcvUzpiHH4`&dh?WL1>0IV_q{zpH)z2~ zBe&rJ+s<1zuRr=Q%uLprS)cy=k z^xLSBb6^w-vrAAQGOT@p#JP$b6DfeHBq*n;773kKtP{^hoM+J-Z4#RCXez>f*R^G9 z$s-5!lu$ks=AS7EG6Rs3G*o4W;?G=}0BPi@H3h!RElOpqat9h{ed+khy*oMQ!L0I^ z-MyK>XY-xnPQp^OJdv`BmA2=_Z^iy_(a7s*i)j*ti}Ahhg4Lwxx6}iCVRNSV&J>lo zWn7=R%sg7-e$oAmXQ#KO5tAeA3recMmDOmLw>~y_m4BVdop#u{Cq&glDLj6vhR4=$k8oEm&MuLT*`a%xz*dz!qw}&N@4X1J za;VbjzaguaoaD3IR&r168W@}foDNoq-#2PX%%#yOrB<|WXPC3wQ~Y?l=3#DLvShh4jO#$;_}kjDNxCrBxB^$PtS`C}8wa9^&ZNx?kOJM3ro! zYP9(L@hV;ebrtg~7H^n3l~JFO)nsu0au+X&%rBa?9b})D+sjY!9!_McU^@pK9`=>a#oyS!gquax*#aG%eJF$y*j~ZNmPM5MnqJ zcw=p{m@}(?OXF-lt-_OY|NU+x<+3d?&3Duy%j4tSx$pYVpg41Jm@}uUAAX$G>Q(Q< z-SYYLg%Cb_rjAWyWEmB|JhN*Zyuc2@>u2(>*C*gaMcW>kyK!3{!W-@XH}^kCqZ*AFeN4-=L z()sNjlhx$=(m>q{?=ha{barO`dB6|0x86k6jDJ|>9keu3Dn!{K>HJrg81frEP57`@ zXFvvG4xd{|$x{5ZlCF>33AwEs`iSgCldJ#0yBQUi&_K+m^N$})E040Qtzpr9B-baf zaY-ca!8l5wH8T8|jQ&LDs%SCuis|_JYsP0NwT20RmNNud^6TWyH~ycF!mr6Z7QBAj z(T;{;Pgs#}B)^J3w_36$GO4lDVrAh*O{Arg{(wPNV^xEc{{PnR&YdpBnz5?}@7zIG zsW_EjbmRc{G6<`%7==VmAwXzklsM#h@xgP2>kU_8AgNj6rdq+&;PcKh-FyLroL(N$j=w~XTIT-%P9nP09Ou}gY|dZ=K+U>tdC3$!G3Szy zfz#ZwaoihwUMyYwuy08o>nMt`+BZ%-h|T0a?GK_*S-v{1*or9~&*#?fM$g}>txQ#_ zQVQFrHh7HsQv9XDD#d6rq5MeLG0eRSS zatfjQC=1dr22}z znKU^q7HnP!MPs(3dzxtAf~%fhNRQmR6-8a)Ml}SQhyl*})_!#)0sqaz>LPq$h5KY{ zi>vy1#r+aEu%`o*V6D@c6KnLR|hJI&Hs1kV@bHx{qqxM9>yM%m)so@~XtJRC| zT-9*r#Lm8wlao^e0D~*NBgDO7)Fyia zSWSs076JB!?ZwIk+-B8rv>LS-%M|W2gA>=U+V2P*I@jpc?VP=pY9dFYR$f}|);a$& z+`2nB5LTtSM6t_%Rvo=oNv;FiP2k(*TBBMFob(!bquBE52=w+RUA3ohPYjyxch-P+ zTeUDX>8NAKMUVWv$J;-E1g`kXt!=ukEK)v!sqD0yzSF+pe)*T-vCeOu>PShdIhO1h z5W*1lCM}6_`c;9D;WfT<^AuflM%c$gxOksHc(#|tV$C;=GZ{21K*!Aqh zb71>^TQu#Hegk+^afSqxFBHRV<~vc^Q{(V^D>vikUH2oPJW15-=Zfzz!x)d(B}=KhcB;*m z=KUOcxKee^4YHAojNGB zDA;KRUR}&r%Tg@&b$aA4aw6YFo{OR~>n{gw{8^~Fx)fb$MRj!Xt7cn6unPpU;5wPA zGXlHy&tdm2)N%F8?j=sj1x@RO;x4t$z-v_<;++{VC&|QjLcyGDcMN^wyDf({i|$s$61odWYxAhg?XT) z{@50*X##h|Ase?G^)@Q$s^&xH#_!=PjYRM{NViGwq-5`)hY~aHYzs9dXD*9HzzC3Jca0a z_eNDY&MqY3OY1D0=xW_0ako6kO?Cj~16)yyF`1;japYP^hX7cE;9}9o;qCCyLz?1b zAFvf$$LmRi&h)&M4{ABqVh*V6LuBK4P)4_swEmJ?vbEzR+2M`5^hq|*)w{nr235M= zvMc)J`Od}O=r0BFiBx(e4P@60VvL>8$Cl|Yp=wSxn{{&oE?r6AaU%F-{Q?GwY0pY^4t}i&WauGR$3@`G_ybZ7*x6kBYRw?1Tid zDS1M(@{~-2&kLBm0PKFVtWIxMNB5|SF8-7 zqXx%MxYBuF4dZ3k>Zh>4lhn_QjWLHKt$mB#D_XoaB7WOWsK(1KmMh@h1YE6~A+r;$O5z zyjKJkeHnyH*S`%4Y}Djc|1Iiy*mcD{uqoZ%HA@ghacB@Ydt%ZMLDmq<;_wfG&gyN@SU?Diu%2sqH>5B4itW;3T$wTYzU zP-u{B3#O>uN`8Y|W1q7B$9F&aAat#!b+7(Od-A^ie-LkTXU8cuvi#!q^87Hm zbAkJz@Sz~Sh5Q9r#yTiCm>+=Ld5!z3K{Zg9`(UoS4`Jt9IQQ^4>u5c_(6QjSmEY+a zHc3CG@8KO6e!H&wm6P%%29lnoow}!T*iGLWl^7w@qEV`Q)^76UBJmn_K@n53?ny()GUGh@@N3P+yf5+~-zdT;E0&N?j-r{~;11RLYzaPRYG8<|#Fg%${qDIVH zVtzUGFRLCjOns}5_~=siUlSf78BHi}xMSG4=hSHyY}*#~zj!Wthi zCYcN7^A*|g=D_6rB~tMV*Ti4;9j1hvAI#<-U=;DZM;**{vV~g*5%nG^G2;__XQ0S8 z`{}w9D`jG+*)MG3A=OU#tx zWIeB$GxOVhbJ3JM6YwPHq0RVYH+ODvWk#_Kjfp=gwrlusBVl+I3}dP!xZwS{RXr zUTY~u)Q7jXJcz5cf~DO#ouR8eR^NmZM5`RCRE-%Zvt2{a!yku-1;4k+?y(-+@S*F~ zm6vSW#dALt8k`CvKn;rL_0g8hotImmm{D2GEXHRbJ1veCrmymUbh;J-)Dk01zlu689(o{ zF`kEqd4x*pxFm@$Fb4~_(*xOC%ujV?24o?=*Qv;z-Y-32~=Cnuq-kjE|b9;lf7jB2kv$l_X(7zRPK`LVT@ZQv_2KP%RZ#_bk zOnx~EJ2TWJL_P`YTFmJr=y-jPQ7NHtGxiwc!bNurladvuQNf`q6m&*Ht6;;_KGZxv z5_!F%{S#RuAAJ$d-$eJvt&H-M0O^fXiUCEC+YkQm=rF@vZ!mb>(XV z@yjGJzZ28Eg0`#F^-N#$a{nErBZ%4Czqk*tKfGy(k~br~Af5O&TS_l} z;97r)z@&bqs#B}#R+z;fBFldgVDS~W%<1-FoxybsW0SPB?x-ZU1$&4_ry?>HQT0`E z&KOMwSSFr*E?sA#vmdwrc<@Enq({~LdmOia%Hy3mWm=NUR3=l5)ge{sc-;>I7_wb++yBzKsKRz^`T^`r8ZVDa+r%KBq&$S zVnPTPbV1#31{{^q=*3}jd04%C<#%JWwl0a!&02qju&tcXG`mT9#6iyzG_RO$^l`ny zc|VQW21MJ8b63MP>4<3NQ9yy+%9J3DeCMqA&9PZ90@jS5O~|7}0FlQ=)u-t#_${*v zry!ptiF~RoDQdtG9+i>MxWK!5QL6|WDlu z#x()^X7rR%^sunq(6&KJf7y{Lgsk*Ro(qykbI9>vK^3$BngB7XyBH5T%c;r>vY#xM z_%r5)hmUl-%{X{3bJV{zseuT5`cVuG&YsxZm|zZ0 zZB@Ez=PuQ4suIAXD|B_&QO3y7O}~q74Kw`rxShUc9BhlF9O(kPR9K@mww1+Q8kWf{q)|{2AD_q4rMvO%(d;`ZAs3(xx8pGmqE{$q zo;tJsnRZF|w-N!6S#`y{EN%KOso-t|{G_1p6v!RAB{Syz7uT|Gw4 zOQ~K+Qsg#W6g+VTEfrub2}_n_SSaR_D}(5S&=ZU z8m{iv6jq2u{18yOl-dq%V@`-ZYu5M06mcJZg~7BA*s$Q?^M8PbvTh@>RsHOm>sIm~zJNW^tnO1xO=4-&TD+FBCUa8c?JaV2DX#%a5kFtI z^ZtlTqyRK@%2`zQu-~D52vB+(h-U!>Y(*<^8&tW}AXZsfb$pV?3iQUkRGwn zNy+zi68Gj-BS62zLOzC?*?kG3eoOc`%5b)t!av)0uH&P5?_F)Sp_s2tLuc(p@K?!u zs~zzzP5*25^#6kg{(nG;is<)u+Q;*EG^+*?Bi23{3Li5*{b|N!{eSWjou69hev_KN zOSJz%y_lQo_D-9S}gweCHvLe|5zvoc$8Wx1-8{*TzR62>i4;NT6^o+M(>qrSE0??+!_IqlqhD zCw@>QEuHW07bm}}wtw2@iAZCVyr!~&=8tAAT)NHUe~@#*)lTPdB^?M*EcVH zaV@tw_22(K*Zt?ezy0a%*@k}|4Y=H;)JV43!Zg@YiZ$;oEaPQW(+C;8l?IB33a6yP zDNOacn^%vxpH`T-JA~`HB|K%4jwuf1pM=BKv` z>KXgCT2U4KGSwrZQX8$n1m#zENFO{++kX3jW>9EH)Z=!u@*+qvY%g|YCO8h&*hTch>a z%=2VgfUEjtv>HOD{Wd9 z|CcAto(x+UG+4tgR#phV#&2ITYe|7k?&Bn_ufpzdYeHkKldHJ*sosi|iat7s?9m&H zic&wQf7rEtmj;S}Qcg?bzDG|e#Aey0JlT|3^SDxdhix6d&-3P)*w77?rLcZClndnk z#EWN13UUlPbI36qA{#ktZQ$KuugdYNg+3RUy=@d9xA{n_K) zrd!jB#nau#1hDM1RMe)cf>ATFKkGl3&G1G;4+| zq!0Sq2(wS^9FAwV9=VW(qda1q>xfe4olm+-ao@6OP|Op}FJ-^ojaNR49NW7K?iiIZ8~uoRZ+g}t~{XmfvBGS4IZB3QMmG6 zHa-V_XqW?3O|52vVM~J3;d*(Uwt)_yJ)>hUI0l`FRR$Mg!yU7PPuE63m`j29bK2!STsB zbwgqy%$qmfCMe^{KUZK;J~1;JuTF(DA%eD(`!iRrC%)ZCjNhYnu9wPxfBrah*WytN z?<5cINwb1l0+!0C02WNt;FI7KN=(s*^S$g0hA*=^-wdIaYB=(Ri(C07TTj|8>N7K= z=4USkk$#%L?M?ofXfYtU_(HL`55Ij*T9Mb;xHA)OKEj*Ok$h=u8YqOVqq_x`CBFi* z$#0>J81(6)6oHKuYGUZk)up6ZqQUUPi{!s3?k~F;@Vt88g(xeG)JMshNcHAjv6`{R zxQ}Z;?n9YJ4Qzbc+Hv0=+hZcQ_9}?FsEo<5nZSAKLxF8#isyE!{>p{2IBeq6k{FmGWXe@^EpYQwscU>q3P1xQ zY_HEI-kIE{-*ChCgjwkB-U#pvX^XysCgOczuaa|1h{&Y&I+xJL#HiTujm!2IVJ$DH z7X4!k3(seTv5mTBB|3obyz1K0a|605dQBilii(3mrSMG&wI2;aCX(h zjqzc6ro^mG!*UsQP5#j(q2vE$0bF$4KWKO@k{hY~b1zwatR^tL0BTMi3}G)zc#l^% zhPO22Ra5>INE>Oh!e7ruUd22+EP1TAN;OwV(PcrqUn7&I}u7P(p{(`hh|B zY;YiDxYj-*3-`bz{4Igl(!5UH$MO5Z#4XZODeH}yL%h(2?>p|Y^EnPo^C(RI5`eGk z`fi}?Lw6bSvjauYLae@Px%$)saI)OiR&6k>?CzC1z)$pX5N!Jb4nCM|5%kGoSZ*W}tDJg=yKY7G0anl~;y;pQ8(EBo^i=5&Lr0$W#wukf?UcJYP; z_9#tZs~d$HME1_8edTJ_R(sLzE$BuYnvPMLIM@>|u?^pM5>~9h1)nHrb^1?R$$5-j zAn(5!8F%yy-S6K6Z0ib;oz{SjC|Y!fW5!y0?tC_-{n&pe_jVY4Li03~qyTUy&1pW{ z*<;;^U~eCh((qb(gfbIc`S<_}Z2ZU!o-%My%$4$XfsH z=L^67@UPAH4+qTu>$Qco@5X>7we$CNw|>+3yEW&=hXGB4Y+QaM!!%geN3DNo@1x`= zw(^wBtdR%+aX(Q134@WjGv8at;wY)B2NJViE(97S#c}`I=EGxs9QE?myKijI!FdU< zH%iVX2GU-n{8jI2 zYH8M$`c_-t3atL&aCgs1?{Qj;#)CU>sE(N1|93yqEcDU~Z4_6+lyHBIo9Ddz!|xw7 ztDQf>FCuwAFYFwv59K||aT_SAB{d7~7Ht$fv-1WZ3u)FaU?7ZpuJO6!Mg4gHag5u5 zzTg>1_S`LRJj+dsNoaepvaeINQJkTFDDf(6YRsHgRD{;s8^_cJp(D)pwe>aeK_ zrm`heyS%;9?w3k0NyB^=7;nIPand}(xW$BGp`qSf+X3Ht57E&uVp9@W6_qqH+Oi<< z>3v{dz5JMyC(i=sa-gU(!)05o>zah`|DrZ!IVvfPg%0$*R8TQFbcVG0*zf@YlCC_S z2&8tUb=-u(RAV05Ap(jma=*AMn@cng4w`SaBVvBH6R#iP{v<@bNY(N$pwYAc6U8c>c4xP1mUsqt@v;z3NU%Avz&y5Kpz|919Y=YRTsBsD!vv>phKI z6b^pmgV=8iNW;VGGa$occy(sv{7B72PjjLr?S4*Ln)cYAz!FN7kjUO%zhJdG7d*wg zebmG&3~&GdBu*`9q=?fd)?WkfMca0}fH!gBt3S_3p@T{Y_4KGahzy-T>Y4hiS4c4M zEp^XjN*ZqM2`doT)RIFxLUo5do9YtCGp~Fo#v$I64)5iHObaWSa;L^mI&nw^9|Tor zq+t4 zCT$fzvAD7_niv6#{|HrC;T}lLnLl)V0v5BDxTGot4{A@WB=;-L-`Py))i(uW0cwAH%_E)P{wx=u+3C`7dA^GDlXc}o$+=;nw zPhNer@~;Gk0+fc6-|qfV$~_^U8+5@xGh+8u`t&0-VJxDn+87QeyzuMdJGU~~^-fx^ zE_=i68p!WUS>xM>1{UYWZpNF`9dAjW{@;I*g+8up$=-YIkd&x1m8L*?$0#X9|3G@J zJ~*zEfsH*6{Fzs7QrJ&qkyrM{P4&IQ z{t!EYr*1k&4X@ptE@>EMuYC*l)fQ>io$IrAVjk85$s=>f{LMJ~SeF1$DDhuYC=l}B zWjhCcxiHCl!qUEjW&1vcW@~RvReVz!tx4rTtM69Shwg0t1LK`?i9{qVZ!n&&k$ALV z$Q+MXLI6XtuSQq?hEe&QaU>mj$i!u3GrHlBr@C~sYAVS0PIdnL4SUa#mRT#1 zSp?8zDPZ~b$Q*f|TT%NS|37@adtB0YAOF2(%e6`>S6C`+vvkc8y_{#jZEd+OO;(m# z9`MlQTAFyk6DL=e`miS!+!o?>lB~2Z*&I0RM>t$lIv>Wo2nPLxTEec2v7f{%xVjLa4FkUznd}a4 zRFA^px@j7x-|vMs`bo_nuYra;KNAl`d>Q>}&9Ms-I@l8AY zl`@@l&y6%&yb*d}l91=oA0>mubf#`SRW-PlE}Bs@tc(p;`nZ;DN6}2wcoCrPdoOHi z6c^Fy$)PnNR=CnJX1|d|5V|VfVPAsXIf{Et?gCam;wuAfykLKD>?+0+B|#jLNPHVX ztOEvY5ABRu#%P4Mp2%jJ>$8(w`uCZmoUQYn5vIt(B@Omhp2ROTxl~yL6u%|>RkMOz zkJAdjP3>5d)}C!z?X#{jFfS&zSY}%mVjQ^=a?i^O2|aZ_Xo>Z|zVLN3JCDDWT1+T6 zM66}kd%lRx$ZSYSwJKxQi@aqEvZCg98CTe??O&ONt#k?pJD+K@Oc@~FbwhdCKeg4fBwd;iN5-)3km7&I7v|k77fp!%Jg$a@YZd_} zG@Y0NXm9okUUFP^i(d(#T3~};|v%aoiyH5S=7O!l79D!LlG z%QHR(@#aI#;&!5tRJP6Jj+TxueZs`)eO(V@q4U_@8Qe~XOlQ9+Wd7LY{tyByx{}EN z(D+6bq30ICYAeklRi=^O8Yj?4C-abInlegCE96YPIHd7;}R76_k2T!jt%fcU^RQYHnpOf$`IG zE$IYjo%&qs_n=HHP;wRxf*P1T2@VV`s2uv(ZK zRI>}$l%T4qUW=O4e`4dPQHTTh(mFTOjGp6Bv&Y3!GtCc=!^6E*=e;xL0^ta48Y058 z7qEBAv}I}Ua_7Juw}@@s>2)7JO_Jux1(0}*;;}oRVYz1d<3FzSx-#`|M0K!#oD4-N zoUXWw@DNjB&!QoPom!#sEbyU^frzoQdrIoY#c|xAgS1WVZ;+whApojLx(M-Wo%baL zH84FG@SW_U0z_-D9o8qoi#J@x`OW^FNhS3e)0zQ`Pt+Y>W>*s*j1Y*6dyKX5Md>YQ$Kk;vXvm!${(Y*#LW2VuoLkH_q7ZA#QPr>ytTxOXc8VhgHWg5>n8Ox z!cX$g!>6fQU9;xM4;>AHCdeva^a&1=4^94*dV%IB|`CW z--LNQ7G3oXgp4>6%-#tcyorucw_O@wL&^t8j}l;iOT>)OW%r{wW%+}^Kum3yc9b{zw--yL?9xfcA)f0`O1_SB7e+ zzXmDN;QvbRJ%G%t^;R&5Nq0?W)_{zaU~uC6j^ICG01I5!_h^cOofge}uU41Wbxa&g zoA|1QR;`fhx;JP)ggR=pxIcMJTGT43I9Z4L#KP37EV*lp`QzpogY&5=MbD{0?VJ$* z#qR^=F-Bg=9V^tjWU~^~DYYFQXP|6ONs_h+12VeE%glDMpv=v?JL0oGSD7Rc)s z-aG%y6vu`~w-sMAFTvOnKm&Q_3{lyaEF)Fu^W)ema9!m1T1!I#Mhyvv3v$noxHWKL zlj76H@uolCes%HOtnywNcJ3l#BBIT5N#ht3wJ!*Q-PP{^d2f|%=3s`?Ed~X*JrLQ( z%mx4DP*#X95!y+b89|H^kNj4U8Plaoe=8dba9C=NJ z@Qi_MRafi=5f;dUrX*Z2V__Dn(&t=hipXhu!JIjojU>FRA|1z_$vXfU+R@RmdQR)I zkZ|-VJmsNuqIVXUmeN!g7p{1gm!{wcbcvP+PR8y>ph;PRg( zKk_D0$(+fL@jT zEgs0Xz>6N5Q-%+4_7_d{=M}@ewMil-sE<63u2njtNM{j26|58JQZBxhV75Pl747eX zJXhDr3*zzq1&5pHt6Dx{k&-O_U8L^f$b8&6B)pNk5lDX*b>yvKC> zOEIt^*b+=G=*sr~vZN#_9%%(QW$?IzDaJYFodA2jhg`Q=2%y$9eTNZDgvU6{3bXgdrc`i-lnPycQmZ@WTF)o)@9;HPKzjD{j z2bhVCJEAWf+E#^JUP3zihFSGJgnQ_;lc$E$mtQ%uPvfpVpjnhB-zw~+cTG^K_nd9R zQ_{U}IZ^Lj`zv~Ce#vw}+2auIkzMDIs-URZ6l_a+%W&!{pUW~DkuqnMud};-p_vYk-E7E|bn8L%@ffI(Y(|gw77qXg7o3TG@yKg%&bHX;=mu0~GtvAo`JzWQ^#EIWJK*Vp( zXy0bIQ|e2HVHf)jWrL-pPsBpE!gkX}$i_{Gdjf-($ME)l(Rx%#UC+DpGiXD+f-FHW zFux!^9J3qWxVA#r-$%Y4Eq&Hj19+X{j!QJUCxomfCoyvu@C;G-!Oeo0u3z^o3pUi| zNZ8!6J@xXh{Km1&>d?p>Tz=(8GjRv>&d&5Ks;^n(XNPaq9s75EgyZqgd^p{a-yfq` zei)8Do10TT?qd3aoeT6qf06E9InRu*O}cSveO~#_Q7!luToRjn*XQ-3izQf-QG=$gyQ+j32Ma&X&xyg+?si@Epyvsn#St=WhIHN-7!;3MlCE}U3+WTmOD>k-bF#9^s7txFZLllomR*7&UP`P0fzQN57@i4@6ytS z14lW#%)COtk4txF$~I>7f7VTHm}aO4&z(EQh)&L!FwT=A>k_kb>~L0VW5?Y@`OgdS zOFRhPrR$QN)q%?9&MO}nglR+M+9!aaTo~b{M_v{Rm%<%j5{y{;smF;Hk*uERWBACq zfI|0Cu{oJMd(z*vW2QLrvFsEPKGrjWm0cxO7Eu`R*J)?Y82wB#&SlXF*S!^+muawD zHj&!zT}LpDNgllOkcqP}vCohaXjh>&wjhKp1(OM(Z0t0rdzn(+W+3$)Hq(!kYp3fr zLMp_g;eMVrC&NP1+4fdpKl$WN5SOKonc44TQTInYNR9K#R}LfC;QW}47WD(V5#{#O znU`r$JU*NDX}tyJsz;AoiDU<~GY;mi+Cepzs1_}pwkXnNe@}ffhg}C)OgOA3ib4oV zVV2#Na(8s!FMR72c_7eo;%KdaT9F~p>Ep~Q0g{o$bW!#J3YzH1&3jTK-(dGYzR2Tt z@!=I7W^lNh`erzCH(bBLb6YcR|JV)f8CzI0CIz9pw1?06w4XaLvUL(;xZ@(%N(X7G z`$3BFoXyHa{#PK5ZV9Iut=V95c@&KJwFi&^&l?EOT)P@;Z&x~OT18MK^!fsgNzgbO zC&R@(<*-8bX_q8{u4M>{E)=E0f- zw~szEVgvhYKcZ%Anz!FGT`~S^Y;tpdq%D4t^`xca1G@mczSvX7`Yi;BmadX@%dGm5 z)fu26XG$>+WXFMbH8f|A1jTmDqYM(1b!fQZkAv?Sz1r&#B|7^764&`B1o>K7L&9$y zy*kZ!ru%Xa*zAaADz+Z7BDx|h&fnON|_=SBLw)ocP z0;O(!s)iEYur(s)J3ghZcHSkYRm=8lZPYe!{sE=hVYlAn_w(uZQId)EP~2hn#6uy_ zXxYKyPW-#}em!Wx>^_8peVZ>?s!YR+vRUt7`;89TuOw~I{_aY;u#BOxqzelBLGr{R z+|;!Gr<3dQi4ojPx4vI3EL`BxO<9K<+7@YTxRsRGo9f$JsK2Hgl+4Kt(UiOTt9pE| z6coH3K8m{l7r#&L7k1tGtfZ*p#dEsoP`m14Lkf!2Pewv9_5^vIylv|==5PLtCZWu# zWaYiK#;`5@6SCClw7!SFJB-<#~KFoA37-S->SdlTXKKY>_W+Y(9bS>GKpF-xd{&g?~L2m`ZjIH(r$3c6Tm)= zY7}olw+yLY8@mE>t99#GWRt>|4ZSwvj%{bs|GPPfa8U10Z&CyyiHW1k?u-0rUE{rL zg}gXzLZQ#T7X5Qmu=kFeGCOw~>AixXRwD<>nrd(*8d2W6*vcNf};ajGkDOS5L z#aXzfJZ!f}7>ii;n*#&9kw17=a{R!rffIS7q1I@fFB+OOr^;Eu?^2oYHIn$P@Bq5< z?&-|{NJxh24ktB3YK!AWK|{ntG7Ro)}`)%ZF*ZI{6-x5z$}@T4Y4 z{ml@!FslM=h^n2q)8bwx9oOAE5YV?czL!Cmd9|;VN;6H?n!h$Mt9QG+W5rOff6DTb{Qfo2o#s_AA z-=#pb!rkUUPLJ~rpd8NrZiIxb zvObTl3aJg%E?5itjbT1O3VzWAobo_5KkyI)p5)$y_y;G=@c#oQK4>ru?F2OtD3LLg z44rk!t}9%MN!M)EeXy1Wwj}|X*yMvDQ>@jR%)I`x z-bg@zgW`>KRYQfckou^BtnaWLW2oO#lK*6$vP}%ETNbD;!zi5bE1-Qu;ltC=nC3e_ zeh~4;1TQ}0lg|-mPj4$$@Y&)D{0e28(@qY1V@3>(dQe%K?fi{;JQOYr+~e12-(Xna z*>zs7UxKCmoou_fM1zjm*RQZxHilklJ^?I({mx%sTHidkn5h{<5=!=f@Z_L56~lqQ zoo4rVLj9z(W9>z9mkqG=S`Z|DT)rd39d0xs3kPS8Jg(U7K=@|f7zKKX24%+iSwOMh z`Lvo7HR}6_N_-C&rCvCl8P>Houd;mZPnt6@kD?_oq_x^x0-J|g*2|f#u|3YjZ(>?M zs=9r7fUkRxD{?cgqu|oJnryxdUNM3_R^LeInl-Y%^$n3n0!_A>MFPX z*9}a}*?&iDUi*UMa0ae$K%iS3s%({6&vOj1S2~v1OaPr3Vc&ommW~aXy5XmU0$k4| z1;5Z?_`u(4YTrZ1aI=yP1FE1QqNTfvoc2v2V6?1Dyuhpm#Cf!J&EEpo!#{ap?bdCw zvM;nwxGM7-wDIzWYb^7^{l~tu<#X61B{>5x97E%|O2372Lem=OHXwj#NH}1zk-j`I zvrltv^9f^vTRDSbh@=rD?bn}Wwp`0CLF#}sl{3?l;8Gx-ZPIE-L}(6 z%~;~!uSX4_V*AThf3L}0iyDX|40^{q^Er*~6C$^MYPom_fgJopb;lPj&VeBDyQGGf zV>NidTk$UriXQT^IGB^SyqTi8!FtdTeC?)e$ar9bG-ILgv5Tj)gZ);(&J<%^w6F8{ zwMLTf`$P7y&?aq7E2yh+eYgnb-x4LLl!bToqRu7k-?&C|eQ8XDvSPq$n+)12;tz0yl#b=?WA(PGrns8K@2hJ{IP@|#p8MM z=d}|bPE#Ca>4J*_AZFeKdozC&GodeC-_SlV04;fK zQQWXHss=5b0D4blWH%69p;3qI$OQC2m_O?!Db|RXA7@t}Vx9d6|3PO@Cco;ZyfPCd zhXO`RNQT^jpbpI{cZ@Pk|0rEEaj?cEj+>{t_Qc;u(lL>JGb^~H{xt9&X*WSsiyZM`r`tlQLMj{dz{X=rk$L8}sGJ-bJ9>WUL z^YW@ehiTXd5oe?$gLIE~*)@CpG0Bs2>&_)By;fX|Y*O9`NFE+v#}6My!zbNR z^@g>cMnpAv83OZH3^pX&SC6zxXqOo)=g^+yKMaFLdCEmIzA412j1tJc(5WJ)OerNbTf!_=cz8=*us+<=?D0YOY%#VP*cOZZ<)c<6dIlNp2RSbtCW%kEwl z@{Zc&IXw{9{8&s^v(GJ8*ukB8FPOeF*>EH!g@Z8ZE=Mwke|~$Eo^p2dY@pKRUcJd> zC~kN6-7m7(4~qyn%*Oi`3+NPIG>dQb$dX?9+KH}wac+?^Ltv^@4oqkaFNI2dK^ zaC2n4_b>NJ5>X>6Y&M-yJd*tZ9s4bR=6=$_qB;KNyva(It2cgCx9T>oqLrXM&`X9P zaq3P6p*sM-6h}>5djmjSQ`%T$yr#`VjfcBk9Kn_OfRX}pF>j`lp2UXjEyr~7 zBi`S3u~ak)oR2*@34anq2p`{EF_p3de|J5v6@p&p6)kC zxl)@XcTdSVkZ_j}Ai$2Ys+o4CONWgMVH4zv>8F)NI_A@n27)UVnW;T&@7G)^NF<$4 z#->BhbR>SPd9;eYPTA(}hMZ}t)7{}&_nv-1CFe3cOO@UIVAuQ>%=}KbI7MV)Z}eeO zEjScg>b?l9ZL+YET+toM>m(2{s1s+18m{^ye2C*N`z4sj`WG+6N!)-u(=;^p#VGFm z<3^x2g~c2yz?PDeVt^p~GTSQHa^u zQcPKhLi*YtF4(AjJ|YG}3Nh}=Lkv$buy0CYnor;-KR}f0Zn48pt(|om%$qr?sSN>Z zHhdse=cmV_`4t5X&_TDE7-*!r2lP7Kz@vrE_5_c%d+(e#ekq zU8b2fd#C%(qT?%&u(#-rGw)kZ1vT|lh~{GpaVgS@$@!$2n(50um_Gc97jPr#K~ZvU zh?#if?Dq2L4=cUs>*iWO%UVj)TFXJy#3LRc6^<>bEilWj$8Q)G+-zZP7bxc$FOK`` zRO_GAmKT;oV2l1eN{K9Tf-TQ@GoShDbCJ#`Gih!U!XMFYRpLMVcjZh|Z)zqEE%>dN zx)Ar_fxiP@$wYi;0wl=_jtZ-?YD~pHpzbEcLQpPwNNd&*(m^Rl+l$T@Z&fpQ0KNBG z7i!1zjHiA{5q%-CNx?o>A)ULh(klqDa{3(9hBp(7ZzG!)YH{S=>hM0BKuY_nBYtSw z1duA7JNqF7JUE(9^$H@E?V;cmv;F%*D7}9^cPJMJ$dhahpTjC(@SCPSDOW^J{e({- zkjSOT{HRx{+?@{@WLEUKh_dN-N!8Sn0G$q;^s!$CyewYfJWGzhh7+*EQ&-7Ix>()7 zXtxvJq)jV;&-4vhsi9|e)w5x1!0L#0!0Q)lM8a8LA z#RLllnTF*$lNEQK5@)ZYy)+KFf~=uuU%@{Kt)1~BeAV(Bm!dd9Nt)HL5VvL~d-<2v zV=@xs=W>C zz4o&;bh)e$1pvjU-!2Nkk+<47|9h)d^4lAb?ZJi)UuN=YM$>YqzGK8NZmis|Y294_hUZnuP;2MN;rE7C2!Cn9S{)@Mf;npD- z551p0goi%uLp+vJG;dFn>PvW;yfkYzc2yn%PF%Z3C)P}4tz1Jv&FFXZB4v?^pVa`ibCo@*b7GHw$)AI-&vPZMwi8BNG~PvzWffOmbc zvtWqpsu693*;7^9-`85Hp4hEb6nEr=ez;V7t=jFNk+H-&ZVVeZ(s#7PJ#to%`yf;I zF)c_auuLPavd;pj*hKZWAP=ggbV!b5X{KmFq;V zR4ck&_M46)7jnLrMTfVuSWG^zd71d4Zq=)ohUdAN8DFR%s2#zWNmq8PrGJU~} zBetRCY^qH7sOAA6A4ZP4{VcCUj)zYSaV&QEbUFt?^05&wN($rRTwCOO;hGi!yAm&= zB7lY96D6$sAre$Jut8<-nt2eJ+NNv@p9odj+lDl~S1i|G8P(TL1+R&}+y(G+dQque zR^0X&X-Vk1kLYM8G3`k^>w_D)%fMnp>F|yEbmNw;a+gUx@y?GyC)2V|84WlON$~QN zl~b`jNGp7W>2}i1Nd80{)Ql5B(NE2g5|Xmt5Ad=R+)vO(ER z3tjB~ny58?ua`zdH@Z_efGX8@)y_!C8xD9|IhRHj2@r!aL&Rj}<=Xo8HuMV8sm9z(dD+am{jKT!UZJq(=rt~$IUW*(ggsd)AJXegDD_uZ z7koEsj4?ao`ZwVP1cn%PdR2J};8~8nU!S~0d&1%{9|oCmy=-Htczt`i$8%Wm$LR_}oh<=8Pv%s5s7Y4Jx_PI|;+t zcw1q94oxi2kkn^}0B4rezI&VY9;CPbiVCyB2@MG;XViatF6TAV*T&2&tMH4-y4B^a zXIUu#?BPC6i~}9#TLJ)W{;@)9l+KFol@ZcRXZYWj*wmto+2Et3l4;c>AE^-|x9+Ul zWJ;I~&ds(_`z?fWuCL2Wp2)-z($8NotJQLkUOX+Ro49st`fw!w$F8Nax0cW5{!MpR zyhCM8|FrB<;m{E?d&dV~I?WWGbZ!3;W$TDv+U{PaG-@9eZ48L^uOD#n(>*97w9Tgv zC!7K1#%J`V-kVU#9K9U3tc|ij%Gy(#mGGH0E)7&t7rZS>YW=mN;)(O;#SMhS(HuyF zA>sqq^p$rmqJOJ=l2IYPpsKWs!+#Vxe|W@s)P)I;F>yFhj6gfI9@X^-_Cpv(xEP^Y z6m1)quDyK%Xu)`QJsbDYi_8)4*fxz#N0u%g$#01i2xWPgGzhU&f6E%Z=_E4dmT{xH z4|?iv&D_a?Tc3<%OQ7pduAFS;{V$LAZdmx-#e(57s>QPIKGHKakym0?UtXtf4%Nog z1j6NiKkDi)c8G&IwkQsHTf+qGc@x<7nxe}!_crP_x->mPb~^a=5@u;1uA?HrBQ?C! zNAb`S{OM^5ptdtOvTj0lJ&|?Nu5(sGy%P+9SjVN~p#K}ENi=t+-xSq#ES%jMi^z6E zF|GRr82zWk;7b`HlJ(}YBkJ>8>wAe_IzCXr|3I-5hP`BP@q zY~OKBBj>`5cmfZ0we;3#w7oLFy0a)$T{acb844P|)@ZwA+&jQyOo9^89r1?LPG}w5 zV@gx!u8xv$buJ}Hoa!GX7)kbfWypD)x>^DNBk1#nu|wY8iK|x;G?&%x-2p>UhbB*QHY%LNhpS4(t)X=+ zjZ5|81y!_7Y)HMH3#XR5ocC?D%FQ^7>`c5~JBz77hY=IB#%-0UX#gw^2U((aH+*Sg zTmGm~u=_Nr|J!Q(i<VFk71PH7LFZSkw8a(;Yu|#X%E?QH%-%HD(v`(}j{^&f`}+8v;L_Rfp8H4Czo_ zZyX?Tx(6d4YZJe$p8BvKmEPLKs}^VHM5|W~!l+7nGlsk99Ig4{X;r|C0@LWFK>Kdb zKxUpTwBwGY5|`3t)XNRStgPjsop(zzslpR0@;_iMRFS}UN%RWx+C{PQbyM%kWx#YR zKs&yz0A?x#>&hUyQmgvRr;ao~&mtN{F?Za#EMHPP$^>JGJlHxBWZG`fUO(xnJ_y^- zwkp|tgwqco@`dAhf*&a>k?+?p_nukM|H8cXdvTgYqV53xsS0YeyJZXpS*!Vk9)8xZ z1m))3{602K#gyDg?YyzxSav@PYPlBms>GywcY{U~cWtqw_i(lM#05%k?Jj4zChACw zR|99^b4Z*8bKb%CCjNqVK@1t7i=w zx9NGqFG%I_M_{c4sdDLWJGyE|Mh0fJf8*1F850l5i%oA>mVb9R`27K~U(ly9bj;HT z{CBT!wOJ)BC1SLb`%{M(3T`5;4jKinx}573@RNQ{gU(h%KHFtQPS(oG%uQ0E&cp%u zMHMz7t~N;;mDtnSv3TBwaqTvB#Zg&p?ZFH1%%Csq8=}x(n>K#VP?DL`|{?@q++3vED--fO=c>d}7 z5YK)NNBeVeyqC`>fX|i12&ebm^0rOxCCK8}9qHFT(z~Z=e6rwMa_HS5ZIg+Wwu*YEdLAGKHQZLQ<=KiAKAv7SO)t)-#j-)iSuA{HXv zd)Ud+6Ph{RVU({qe{YQi1w$j1C#nc({{?+%SKHe0QgMR@0b9hn_RP4Q&=`Xm&i4B+ zNZ)X9a;~r|FbHqdO+ULbdj;qf?phhHC@6#Hc!ecn-P_=qORW90@Hj@^2nC6k>g?R} z9r42Ydhx6e5-Ah}bvULk1m^nKNtE@4;y>ScdA2B#W~IXN=y&KaHJl=@%wiotq)hgk zlx)JmZThR56M2>OPqosyY6t;x0iunUtIZHr*Om|lv$?K-PoO(;o?9ro zJV|TAf?-V5aeN>RUN>Bpm#)E{Uy`HXYXz8L{MjbvyGubG#`LW%VX=Issg>Z-|I$sk zr;HQ6B#i0rG#J*6iY_2E7&A@P8+s2W<7m27(vwN&N=V?wa#0HG=J8c^jaP?7EK#9Q zg#QgWY(0Q(8jF~xHEGLZyAB}enna_l3TtB9k&!X6RIowL#1lp?`Nt4DN0wOv27ga? zxwqx4Cz!~3ZM!Nr#Kx+xWShEIK+ax^5jfGBfL$ zEWMD4n9&JG8#-|qA<2n`Aeo~04che%QUHcfkiA@9gDB_Rsc-h!D+d)0u-*XrK-qFt z`07n&L&;WLw;l;{Jm~%naGgz`c`Y|H+FBV$d;jtU{Gr44?Mz!+>sjWHh))DMDc+kq z^EY--Q%m*OCufZK&euRnBbzxT`650MtIc=Y-UzL0xs&_~o>~8<9oxI#?W8x{9hUa> zyzl|SO3luvj=L}GQshzZGFE?`1MphkHq}r;B6|kLU9fO=vl0h{SCf(gY`F5#%dSQIoZ^T}odyRRw=4bKh;3c`$b#6KgEg9(tAm`++YnY#Qawb= z`HFY_mpDp$I{5Wk-lKGBXTytb6@WPQX%-lTPr6&w3TMvVbw074F5Qv_;aTU4MxH~k zQs3wDUI6QJP|8I&GKZ#htXug!Ced()*7=Ql$(I!_FsD5iUt+!x2YjkIc=J$cuzg2W z>bE2X*DTAfX&!;-#r(;7GI~-o_5c@Zom6mpSd+G`=KH9+7#m*R#02ZtZ2IHxuUnRo z0ky%HUHA5Ml=kio@W(_k^%Up*U=V)wl6}4!KBckyX;3`BU7}w#-3!_jSE2}?Tn@6| z+wLwV^TT_&Nh{^pAHwk)8&^16p}B1zHVl*A*Og)XbwZkgwbqxxYoAG}mf1a01MKLV z0|i!NZ-Kk?QK^)n$u!wDz~t2BH~pr8MlB>RdKl2xww)mknAy6zYHIlH7`z2r2a>$2 zFh<(#x6op*)e)1+1s_E+lVjd^!v|mO=%boPl^{KGPWK`tMD63tg~JJ!lkjfjQo3^Ta{0zU zNzr)_CJ&9wT&l|(3uOrVjKE_|j13W9uzBU_!)p<|J*X?oikA-n{=~$*tjbNJFm`Tq zW@;k1q|#*qzs#c)xXTKTSAEUj5c!R&1orVwZ0k%9lzr92xEE?4Epf00lice*BB-S_+lY^}yYjua&wLp8#DyuL}v<@?^zh#s(XxOKN zrVK-l%dB664Cy`NHP=;&m3B_6`%+`fU)xQs9lhO0n4Vl>%9gj*;f_lf-tH!l*1jLGm@pM7T}@C?wrK>qP-kqi5a%uB$(UEYt@W5uRU1Ar;O{x{tc_+zd9v0#2q&XIvwVxe`%nO_GVkopg_C04V)+iX5;Ss2W`vt7_YR5zssiPwZ9n^Pttmh)R! z^tx|_+%|`n?0#RBq&gP)<;pVKu48llM=>IE4l#8f`^N(AFZ@S4!uxyvnX&C|GTep2 zq6h3)#`0vL@A`9!;)v8T%D?7k)YzTMG z_W)^zZx@vh%XnAfqq_g0di@t!7x^n{LMLc+B?{47Y@5E*v`pS6$=h(9R1}(E`E;t> z*O0_K?{ECTjhq_FPgKFp=?>w+8e?X`%^JKmH*EWG39|M2Ng{UmOvJv87NhY*R%d?L z0n~bAdF)PdilWb^o+5uPC|AiR6e+vn+=U111^|JDk#mvL9a_iEb>ZjH^$w_kaI-k~ zvl+8lGFYMo1cDcysA_cyfeA-58YO9kr>d^7p9ISQ(I)}=z8>GlBf+kol^qM^=Q1X# z19l+ib`NVS@`EK5!9uf9#_34mRSw&QH6^~iULF^jcqORJeev2WH%LkI0c6XtHSPQ< zapHWq8Pv#{_!e}}{>E8U0pG8d?Ua8c=$rcU;cLsPCzC0-HbZUR2wnZkp+Uo*@GR`_ zxit;C)!+){O4m1k<0wM~=I~BJ#%Oar&6u(fN;wrN6_kL{O~#xuXi|Olur8m<$1k}Z zrf*{H!;#6kDf53r8Y=4YklT^%d{&i>?!6I7p%EBw3YYThy58TB;WfpTSEvuFTU94$ zj%%#GOD8GGXM}Nc&BRD1^kX?7(#9`WcyBIf}Xa{=)meJ zNv4ZK{TmJAqqpmj3RkvWT&X^aW&V(|B`0KE4%fAm;oC-**yy6_Da8wOxIA6z6(C!} z3#SYbOH9E@@AOXt%rudr-9a;s6WZ6LJ-+=hPrXJW-%l26g!kq*(hB$4m=TO%P!-Qu z%NuHG4$b-@i4VuM&6x8Y;78JCdI^q$9Tm&^p_hH5jdmL0gJhStDShIPRzB;WArxgQ z7YRzUGU&K~$N+B$CbO_7n()i6;v1*)=tRME>ruVU{B?pPIyop00 z`-lM-;oPU;>dxCE8JoYI-Ujs<-P!Fo+(52Y%fpV^c5hutktgfM^BAYGi(xUZ$pLR3 zG0w{q?*!pf6KnQBI$H6Ds@k@{_=aIVR46tIWzCrMyz7WuwA>*6rAJ~@viLtVLsO#k`12T?>xD3uA4AQawq6{u ztD$;50OJj&3NZ_WTsgZDCd+FZYW%rYd}6!qdhC2O0QCW7C5`xS0tgjHGyv7D@NNt^ zHw2kGOm{8Uxu>}*@`fE~hGECV7E_G;0_vcI4CCMvR5@%qF>ef_YbzpFwtKC=2QnM5VH|=nHgkq!4)cAdf61TCm-J_kUf7)c&Dho z^kQFCsCIvOg7BcG00GEi^1q~QeE@bYbYT@flA0%Of1vi-$MpF7tiC+bnKAz1XpBP5 zwoH45T|8h(hZ}nOro%(a)e}=DD|SX8sHWH;RLbHK>X5pgx9psFHNFD)4&36tpj~jX zx`&jd3+rRB?j3BFH#iv@IdlfAJ?8E4I>tPDB}D7KEdEJZ1Hf3;82MSua_{z(v?KLe zuDwDz03lLw-6dT@mq9PT@`-k)RKP{?jBc|U7(+>gG+^+Ywm2QnbR#ZmDeOe5hoae4 zTZ|up8r_&kf8C;BF`*U^s!;M&__@3+WO)L&q5$RZ&h- z_EH^Oytvl~neP~fk(Q0Rp*xfVF>*P?$Tgz3h>`myA{WEjv`Z5kL-CBn)!g$|&iW`J zjAulbLRB7YLaFeOH$_;&?t954$#)>9&TJ;~KFok1xDHL6<$C_@NH?W3Z9RpwU|a!x zRhU9IUC`Vvh;xw6ihlz|2~NzPVl_c1*e))wfo}RA?j8?$=YDf~IU%5J{c3cx15Npi z<~rDa#riDlpDO%&8q#RDUVA!XreC~61$FTnm{MgFp#O~W4+Ls@{kTTUa_%u|$Nn)^ z)%QcDbj&_x%p5b253VEOdA2?`EAjE)1GZbsH1n)4y}ioLb65iouR)86ufuL`rVYr(3)?-9%x2{vZ31X z3=+-a1THVSD%CTu;``W0yxMk2BORB){8kb}$M9=HA&=={@lCbT^GjM=oMG+Fqy0^@ zH9G)h&Zi!R*4EUh*gyb>P}mDbH+TOtO}$4hO8@nb=`3`*q^tU9Z|)w`$YS43VdaYY zhj8VCG*48&Y0w3$maDiaEG1STHR^`Vkx=LVhpab`N;?0;cz=yk)-RJOD=kf?u`